protected UpdateResponse runUpdate(List<SolrInputDocument> batchToWrite) { try { UpdateResponse result = solr.add(batchToWrite); SolrRecordWriter.incrementCounter(taskId, SolrCounters.class.getName(), SolrCounters.BATCHES_WRITTEN.toString(), 1); SolrRecordWriter.incrementCounter(taskId, SolrCounters.class.getName(), SolrCounters.DOCUMENTS_WRITTEN.toString(), batchToWrite.size()); if (LOG.isDebugEnabled()) { SolrRecordWriter.incrementCounter(taskId, SolrCounters.class.getName(), SolrCounters.BATCH_WRITE_TIME.toString(), result.getElapsedTime()); } return result; } catch (Throwable e) { if (e instanceof Exception) { setBatchWriteException((Exception) e); } else { setBatchWriteException(new Exception(e)); } SolrRecordWriter.incrementCounter(taskId, getClass().getName() + ".errors", e.getClass().getName(), 1); LOG.error("Unable to process batch", e); return null; } }
protected UpdateResponse runUpdate(List<SolrInputDocument> batchToWrite) { try { UpdateResponse result = solr.add(batchToWrite); SolrRecordWriter.incrementCounter(taskId, SolrCounters.class.getName(), SolrCounters.BATCHES_WRITTEN.toString(), 1); SolrRecordWriter.incrementCounter(taskId, SolrCounters.class.getName(), SolrCounters.DOCUMENTS_WRITTEN.toString(), batchToWrite.size()); if (LOG.isDebugEnabled()) { SolrRecordWriter.incrementCounter(taskId, SolrCounters.class.getName(), SolrCounters.BATCH_WRITE_TIME.toString(), result.getElapsedTime()); } return result; } catch (Throwable e) { if (e instanceof Exception) { setBatchWriteException((Exception) e); } else { setBatchWriteException(new Exception(e)); } SolrRecordWriter.incrementCounter(taskId, getClass().getName() + ".errors", e.getClass().getName(), 1); LOG.error("Unable to process batch", e); return null; } }
public synchronized void close(TaskAttemptContext context) throws InterruptedException, SolrServerException, IOException { if (batchPool != null) { context.setStatus("Waiting for batches to complete"); batchPool.shutdown(); while (!batchPool.isTerminated()) { LOG.info(String.format(Locale.ENGLISH, "Waiting for %d items and %d threads to finish executing", batchPool .getQueue().size(), batchPool.getActiveCount())); batchPool.awaitTermination(5, TimeUnit.SECONDS); } } context.setStatus("Committing Solr Phase 1"); solr.commit(true, false); context.setStatus("Optimizing Solr"); int maxSegments = context.getConfiguration().getInt(SolrOutputFormat.SOLR_RECORD_WRITER_MAX_SEGMENTS, 1); LOG.info("Optimizing Solr: forcing merge down to {} segments", maxSegments); long start = System.nanoTime(); solr.optimize(true, false, maxSegments); context.getCounter(SolrCounters.class.getName(), SolrCounters.PHYSICAL_REDUCER_MERGE_TIME.toString()).increment(System.nanoTime() - start); float secs = (System.nanoTime() - start) / (float)(1.0E9); LOG.info("Optimizing Solr: done forcing merge down to {} segments in {} secs", maxSegments, secs); context.setStatus("Committing Solr Phase 2"); solr.commit(true, false); context.setStatus("Shutting down Solr"); solr.close(); }
public synchronized void close(TaskAttemptContext context) throws InterruptedException, SolrServerException, IOException { if (batchPool != null) { context.setStatus("Waiting for batches to complete"); batchPool.shutdown(); while (!batchPool.isTerminated()) { LOG.info(String.format( "Waiting for %d items and %d threads to finish executing", batchPool .getQueue().size(), batchPool.getActiveCount())); batchPool.awaitTermination(5, TimeUnit.SECONDS); } } context.setStatus("Committing Solr Phase 1"); solr.commit(true, false); context.setStatus("Optimizing Solr"); int maxSegments = context.getConfiguration().getInt(SolrOutputFormat.SOLR_RECORD_WRITER_MAX_SEGMENTS, 1); LOG.info("Optimizing Solr: forcing merge down to {} segments", maxSegments); long start = System.currentTimeMillis(); solr.optimize(true, false, maxSegments); context.getCounter(SolrCounters.class.getName(), SolrCounters.PHYSICAL_REDUCER_MERGE_TIME.toString()).increment(System.currentTimeMillis() - start); float secs = (System.currentTimeMillis() - start) / 1000.0f; LOG.info("Optimizing Solr: done forcing merge down to {} segments in {} secs", maxSegments, secs); context.setStatus("Committing Solr Phase 2"); solr.commit(true, false); context.setStatus("Shutting down Solr"); solr.shutdown(); }