private void launchNextTask() { if ( stopped.get() == true ) { // we're stopping, so don't do anything more return; } AtomicBoolean isDone = forestIsDone.get(forest); // we made it to the end, so don't launch anymore tasks if ( isDone.get() == true ) return; long nextStart = start + getBatchSize(); threadPool.execute(new QueryTask(moveMgr, batcher, forest, query, forestBatchNum + 1, nextStart)); } };
synchronized void start(JobTicket ticket) { if ( threadPool != null ) { logger.warn("startJob called more than once"); return; } if ( getBatchSize() <= 0 ) { withBatchSize(1); logger.warn("batchSize should be 1 or greater--setting batchSize to 1"); } jobTicket = ticket; initialize(); for (QueryBatchListener urisReadyListener : urisReadyListeners) { urisReadyListener.initializeListener(this); } jobStartTime = Calendar.getInstance(); started.set(true); if ( query != null ) { startQuerying(); } else { startIterating(); } }
private synchronized void initialize() { if ( threadCountSet == false ) { if ( query != null ) { Forest[] forests = getForestConfig().listForests(); logger.warn("threadCount not set--defaulting to number of forests ({})", forests.length); withThreadCount(forests.length); } else { int hostCount = clientList.get().size(); logger.warn("threadCount not set--defaulting to number of hosts ({})", hostCount); withThreadCount( hostCount ); } // now we've set the threadCount threadCountSet = true; } // If we are iterating and if we have the thread count to 1, we have a single thread acting as both // consumer and producer of the ThreadPoolExecutor queue. Hence, we produce till the maximum and start // consuming and produce again. Since the thread count is 1, there is no worry about thread utilization. if(getThreadCount() == 1) { isSingleThreaded = true; } logger.info("Starting job batchSize={}, threadCount={}, onUrisReady listeners={}, failure listeners={}", getBatchSize(), getThreadCount(), urisReadyListeners.size(), failureListeners.size()); threadPool = new QueryThreadPoolExecutor(getThreadCount(), this); }
try { boolean lastBatch = false; List<String> uriQueue = new ArrayList<>(getBatchSize()); while (iterator.hasNext()) { uriQueue.add(iterator.next()); if(!iterator.hasNext()) lastBatch = true; if (uriQueue.size() == getBatchSize() || !iterator.hasNext()) { final List<String> uris = uriQueue; final boolean finalLastBatch = lastBatch; uriQueue = new ArrayList<>(getBatchSize()); Runnable processBatch = new Runnable() { public void run() {
queryMgr.setPageLength(getBatchSize()); UrisHandle handle = new UrisHandle(); if ( consistentSnapshot == true && serverTimestamp.get() > -1 ) { uris.add( uri ); if ( uris.size() == getBatchSize() ) { if ( uris.size() != getBatchSize() ) {