private static void showdata(Configuration config) { List<Property> props = config.getProperties(); List<DataSource> sources = new ArrayList(); sources.addAll(config.getDataSources()); sources.addAll(config.getDataSources(1)); sources.addAll(config.getDataSources(2)); for (DataSource src : sources) { RecordIterator it = src.getRecords(); while (it.hasNext()) { Record r = it.next(); PrintMatchListener.prettyPrint(r, props); System.out.println(""); } it.close(); } }
/** * Retrieve new records from data sources, and match them to * previously indexed records in the given database. This method * does <em>not</em> index the new records. * @param dbno Which database to match against. * @param matchall If true, all matching records are accepted. If false, * only the single best match for each record is accepted. * @param batch_size The batch size to use. * @since 1.3 */ public void linkRecords(int dbno, Collection<DataSource> sources, boolean matchall, int batch_size) { for (DataSource source : sources) { source.setLogger(logger); Collection<Record> batch = new ArrayList(batch_size); RecordIterator it = source.getRecords(); while (it.hasNext()) { batch.add(it.next()); if (batch.size() == batch_size) { linkBatch(dbno, batch, matchall); batch.clear(); } } it.close(); if (!batch.isEmpty()) linkBatch(dbno, batch, matchall); } endProcessing(); }
it2.close();
/** * Index all new records from the given data sources into the given * database. This method does <em>not</em> do any matching. * @since 1.3 */ public void index(int dbno, Collection<DataSource> sources, int batch_size) { Database thedb = getDB(dbno); int count = 0; for (DataSource source : sources) { source.setLogger(logger); RecordIterator it2 = source.getRecords(); while (it2.hasNext()) { Record record = it2.next(); if (logger.isDebugEnabled()) logger.debug("Indexing record " + record); thedb.index(record); count++; if (count % batch_size == 0) batchReady(batch_size); } it2.close(); } if (count % batch_size == 0) batchReady(count % batch_size); thedb.commit(); }
private static void showdata(Configuration config) { List<Property> props = config.getProperties(); List<DataSource> sources = new ArrayList(); sources.addAll(config.getDataSources()); sources.addAll(config.getDataSources(1)); sources.addAll(config.getDataSources(2)); for (DataSource src : sources) { RecordIterator it = src.getRecords(); while (it.hasNext()) { Record r = it.next(); PrintMatchListener.prettyPrint(r, props); System.out.println(""); } it.close(); } }
/** * Retrieve new records from data sources, and match them to * previously indexed records. This method does <em>not</em> index * the new records. * @param matchall If true, all matching records are accepted. If false, * only the single best match for each record is accepted. * @param batch_size The batch size to use. * @since 1.0 */ public void linkRecords(Collection<DataSource> sources, boolean matchall, int batch_size) { for (DataSource source : sources) { source.setLogger(logger); Collection<Record> batch = new ArrayList(batch_size); RecordIterator it = source.getRecords(); while (it.hasNext()) { batch.add(it.next()); if (batch.size() == batch_size) { linkBatch(batch, matchall); batch.clear(); } } it.close(); if (!batch.isEmpty()) linkBatch(batch, matchall); } endProcessing(); }
/** * Index all new records from the given data sources. This method * does <em>not</em> do any matching. * @since 0.4 */ public void index(Collection<DataSource> sources, int batch_size) { int count = 0; for (DataSource source : sources) { source.setLogger(logger); RecordIterator it2 = source.getRecords(); while (it2.hasNext()) { Record record = it2.next(); database.index(record); count++; if (count % batch_size == 0) batchReady(batch_size); } it2.close(); } if (count % batch_size == 0) batchReady(count % batch_size); database.commit(); }