private synchronized void initialize() { if ( threadCountSet == false ) { if ( query != null ) { Forest[] forests = getForestConfig().listForests(); logger.warn("threadCount not set--defaulting to number of forests ({})", forests.length); withThreadCount(forests.length); } else { int hostCount = clientList.get().size(); logger.warn("threadCount not set--defaulting to number of hosts ({})", hostCount); withThreadCount( hostCount ); } // now we've set the threadCount threadCountSet = true; } // If we are iterating and if we have the thread count to 1, we have a single thread acting as both // consumer and producer of the ThreadPoolExecutor queue. Hence, we produce till the maximum and start // consuming and produce again. Since the thread count is 1, there is no worry about thread utilization. if(getThreadCount() == 1) { isSingleThreaded = true; } logger.info("Starting job batchSize={}, threadCount={}, onUrisReady listeners={}, failure listeners={}", getBatchSize(), getThreadCount(), urisReadyListeners.size(), failureListeners.size()); threadPool = new QueryThreadPoolExecutor(getThreadCount(), this); }
private void retry(QueryEvent queryEvent, boolean callFailListeners) { if ( isStopped() == true ) { logger.warn("Job is now stopped, aborting the retry"); return; } Forest retryForest = null; for ( Forest forest : getForestConfig().listForests() ) { if ( forest.equals(queryEvent.getForest()) ) { // while forest and queryEvent.getForest() have equivalent forest id, // we expect forest to have the currently available host info retryForest = forest; break; } } if ( retryForest == null ) { throw new IllegalStateException("Forest for queryEvent (" + queryEvent.getForest().getForestName() + ") is not in current getForestConfig()"); } // we're obviously not done with this forest forestIsDone.get(retryForest).set(false); retryForestMap.get(retryForest).incrementAndGet(); long start = queryEvent.getForestResultsSoFar() + 1; logger.trace("retryForest {} on retryHost {} at start {}", retryForest.getForestName(), retryForest.getPreferredHost(), start); QueryTask runnable = new QueryTask(getMoveMgr(), this, retryForest, query, queryEvent.getForestBatchNumber(), start, queryEvent.getJobBatchNumber(), callFailListeners); runnable.run(); } /*
private synchronized void startQuerying() { boolean consistentSnapshotFirstQueryHasRun = false; for ( Forest forest : getForestConfig().listForests() ) { QueryTask runnable = new QueryTask(getMoveMgr(), this, forest, query, 1, 1); if ( consistentSnapshot == true && consistentSnapshotFirstQueryHasRun == false ) { // let's run this first time in-line so we'll have the serverTimestamp set // before we launch all the parallel threads runnable.run(); consistentSnapshotFirstQueryHasRun = true; } else { threadPool.execute(runnable); } } }