public void initialize() { if ( initialized == true ) return; synchronized(this) { if ( initialized == true ) return; if ( getBatchSize() <= 0 ) { withBatchSize(1); logger.warn("batchSize should be 1 or greater--setting batchSize to 1"); } if ( transactionSize > 1 ) usingTransactions = true; // if threadCount is negative or 0, use one thread per host if ( getThreadCount() <= 0 ) { withThreadCount( hostInfos.length ); logger.warn("threadCount should be 1 or greater--setting threadCount to number of hosts ({})", hostInfos.length); } // create a thread pool where threads are kept alive for up to one minute of inactivity, // max queue size is threadCount * 3, and callers run tasks past the max queue size threadPool = new CompletableThreadPoolExecutor(getThreadCount(), getThreadCount(), 1, TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(getThreadCount() * 3)); threadPool.allowCoreThreadTimeOut(true); initialized = true; logger.info("threadCount={}", getThreadCount()); logger.info("batchSize={}", getBatchSize()); if ( usingTransactions == true ) logger.info("transactionSize={}", transactionSize); jobStartTime = Calendar.getInstance(); started.set(true); } }
ConcurrentLinkedQueue<Runnable> snapshotQueuedAndExecutingTasks = snapshotQueuedAndExecutingTasks(); try { long duration = unit.convert(timeout, TimeUnit.MILLISECONDS); removeSnapshot();
public void stop() { jobEndTime = Calendar.getInstance(); stopped.set(true); if ( threadPool != null ) threadPool.shutdownNow(); closeAllListeners(); }
if ( threadPool != null ) threadPool.getQueue().drainTo(tasks); for ( Runnable task : tasks ) { if ( task instanceof BatchWriter ) { Runnable fretryWriterTask = (Runnable) threadPool.submit(retryWriterTask); threadPool.replaceTask(writerTask, fretryWriterTask); Runnable fTask = (Runnable) threadPool.submit(task); threadPool.replaceTask(task, fTask);
@Override public WriteBatcher add(DocumentWriteOperation writeOperation) { if ( writeOperation.getUri() == null ) throw new IllegalArgumentException("uri must not be null"); if ( writeOperation.getContent() == null ) throw new IllegalArgumentException("contentHandle must not be null"); initialize(); requireNotStopped(); queue.add(writeOperation); logger.trace("add uri={}", writeOperation.getUri()); // if we have queued batchSize, it's time to flush a batch long recordNum = batchCounter.incrementAndGet(); boolean timeToWriteBatch = (recordNum % getBatchSize()) == 0; if ( timeToWriteBatch ) { BatchWriteSet writeSet = newBatchWriteSet(false); int i=0; for ( ; i < getBatchSize(); i++ ) { DocumentWriteOperation doc = queue.poll(); if ( doc != null ) { writeSet.getWriteSet().add(doc); } else { // strange, there should have been a full batch of docs in the queue... break; } } if ( writeSet.getWriteSet().size() > 0 ) { threadPool.submit( new BatchWriter(writeSet) ); } } return this; }
private void cleanupUnfinishedTransactions(HostInfo host) { Iterator<TransactionInfo> iterator = host.unfinishedTransactions.iterator(); while ( iterator.hasNext() ) { TransactionInfo transactionInfo = iterator.next(); if ( transactionInfo.alive.get() == false ) { iterator.remove(); } else if ( transactionInfo.queuedForCleanup.get() == true ) { // skip this one, it's already queued } else { if ( transactionInfo.inProcess.get() <= 0 ) { if ( transactionInfo.written.get() == true ) { transactionInfo.queuedForCleanup.set(true); threadPool.submit( () -> { if ( completeTransaction(transactionInfo) ) { host.unfinishedTransactions.remove(transactionInfo); } else { // let's try again next cleanup transactionInfo.queuedForCleanup.set(false); } }); } else { iterator.remove(); } } } } }
@Override public boolean awaitCompletion(long timeout, TimeUnit unit) throws InterruptedException { return threadPool.awaitCompletion(timeout, unit); }
public CompletableThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> queue) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, queue, new CompletableRejectedExecutionHandler()); // now that super() has been called we can reference "this" to add it to // the RejectedExecutionHandler ((CompletableRejectedExecutionHandler) getRejectedExecutionHandler()).setThreadPool(this); }