@Override public void beginLoad() throws DaoException { super.beginLoad(); if (labelLoader == null) { labelLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_LABELS.ENTITY_TYPE, WIKIDATA_ENTITY_LABELS.ENTITY_ID, WIKIDATA_ENTITY_LABELS.LANG_ID, WIKIDATA_ENTITY_LABELS.LABEL, }); } if (descLoader == null) { descLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_TYPE, WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_ID, WIKIDATA_ENTITY_DESCRIPTIONS.LANG_ID, WIKIDATA_ENTITY_DESCRIPTIONS.DESCRIPTION, }); } if (aliasLoader == null) { aliasLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_ALIASES.ENTITY_TYPE, WIKIDATA_ENTITY_ALIASES.ENTITY_ID, WIKIDATA_ENTITY_ALIASES.LANG_ID, WIKIDATA_ENTITY_ALIASES.ALIAS }); } properties = new HashMap<Integer, WikidataEntity>(); }
public void close() throws DaoException { endLoad(); } }
/** * Inserts values into the database. * Call this instead of direct sql inserts because the underlying code may optimize the inserts * by creating batch inserts or a cvs that can be directly loaded by the underlying database. * @param values */ protected void insert(Object ... values) throws DaoException{ loader.load(values); }
public FastLoader(WpDataSource ds, TableField[] fields) throws DaoException { this(ds, fields[0].getTable().getName(), getFieldNames(fields)); }
@Override public void endLoad() throws DaoException { if (loader != null) { loader.endLoad(); } LOG.info("creating indexes in {}-create-indexes.sql (this can take some time)", sqlScriptPrefix); executeSqlScriptWithSuffix("-create-indexes.sql"); if (fields != null && fields.length > 0) { wpDs.optimize(fields[0].getTable()); } }
@Override public void beginLoad() throws DaoException { super.beginLoad(); if (labelLoader == null) { labelLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_LABELS.ENTITY_TYPE, WIKIDATA_ENTITY_LABELS.ENTITY_ID, WIKIDATA_ENTITY_LABELS.LANG_ID, WIKIDATA_ENTITY_LABELS.LABEL, }); } if (descLoader == null) { descLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_TYPE, WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_ID, WIKIDATA_ENTITY_DESCRIPTIONS.LANG_ID, WIKIDATA_ENTITY_DESCRIPTIONS.DESCRIPTION, }); } if (aliasLoader == null) { aliasLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_ALIASES.ENTITY_TYPE, WIKIDATA_ENTITY_ALIASES.ENTITY_ID, WIKIDATA_ENTITY_ALIASES.LANG_ID, WIKIDATA_ENTITY_ALIASES.ALIAS }); } properties = new HashMap<Integer, WikidataEntity>(); }
@Override public void save(WikidataEntity entity) throws DaoException { for (Map.Entry<Language, String> entry : entity.getLabels().entrySet()) { labelLoader.load(entity.getType().code, entity.getId(), entry.getKey().getId(), entry.getValue()); } for (Map.Entry<Language, String> entry : entity.getDescriptions().entrySet()) { descLoader.load(entity.getType().code, entity.getId(), entry.getKey().getId(), entry.getValue()); } for (Map.Entry<Language, List<String>> entry : entity.getAliases().entrySet()) { for (String alias : entry.getValue()) { aliasLoader.load(entity.getType().code, entity.getId(), entry.getKey().getId(), alias); } } for (WikidataStatement stmt : entity.getStatements()) { save(stmt); } if (entity.getType() == WikidataEntity.Type.PROPERTY) { synchronized (properties) { properties.put(entity.getId(), entity); } } }
@Override public void endLoad() throws DaoException { if (labelLoader != null) labelLoader.endLoad(); if (descLoader != null) descLoader.endLoad(); if (aliasLoader != null) aliasLoader.endLoad(); labelLoader = null; descLoader = null; aliasLoader = null; super.endLoad(); if (cache != null) { cache.put("wikidata-properties", properties); } wpDs.optimize(WIKIDATA_ENTITY_LABELS); wpDs.optimize(WIKIDATA_ENTITY_ALIASES); wpDs.optimize(WIKIDATA_ENTITY_DESCRIPTIONS); wpDs.optimize(WIKIDATA_STATEMENT); }
@Override public void beginLoad() throws DaoException { executeSqlScriptWithSuffix("-drop-indexes.sql"); executeSqlScriptWithSuffix("-create-tables.sql"); if (fields != null) { loader = new FastLoader(wpDs , fields); } }
@Override public void save(WikidataEntity entity) throws DaoException { for (Map.Entry<Language, String> entry : entity.getLabels().entrySet()) { labelLoader.load(entity.getType().code, entity.getId(), entry.getKey().getId(), entry.getValue()); } for (Map.Entry<Language, String> entry : entity.getDescriptions().entrySet()) { descLoader.load(entity.getType().code, entity.getId(), entry.getKey().getId(), entry.getValue()); } for (Map.Entry<Language, List<String>> entry : entity.getAliases().entrySet()) { for (String alias : entry.getValue()) { aliasLoader.load(entity.getType().code, entity.getId(), entry.getKey().getId(), alias); } } for (WikidataStatement stmt : entity.getStatements()) { save(stmt); } if (entity.getType() == WikidataEntity.Type.PROPERTY) { synchronized (properties) { properties.put(entity.getId(), entity); } } }
@Override public void endLoad() throws DaoException { if (labelLoader != null) labelLoader.endLoad(); if (descLoader != null) descLoader.endLoad(); if (aliasLoader != null) aliasLoader.endLoad(); labelLoader = null; descLoader = null; aliasLoader = null; super.endLoad(); if (cache != null) { cache.put("wikidata-properties", properties); } wpDs.optimize(WIKIDATA_ENTITY_LABELS); wpDs.optimize(WIKIDATA_ENTITY_ALIASES); wpDs.optimize(WIKIDATA_ENTITY_DESCRIPTIONS); wpDs.optimize(WIKIDATA_STATEMENT); }