@Override public void awaitTermination(long timeout, TimeUnit unit, Runnable clean, Runnable dirty) { if (awaitTermination(timeout, unit)) { clean.run(); } else { dirty.run(); } }
public ScriptedFixedThreadsScheduler(final GraphManager graphManager, int targetThreadCount, boolean enforceLimit) { super(graphManager); PronghornStage[][] stageArrays = buildStageGroups(graphManager, targetThreadCount, enforceLimit); /////////////////// // test code for a single thread // threadCount = 1; // PronghornStage[][] stageArrays = new PronghornStage[][]{GraphManager.allStages(graphManager)}; ////////////////// createSchedulers(graphManager, stageArrays); //clean up now before we begin. stageArrays=null; System.gc(); }
if (prevMonitor!=null) { int prodRoot1 = rootId(rbms.stageId , rootsTable, lastKnownRoot); int prodRoot2 = rootId(prevMonitor.stageId , rootsTable, lastKnownRoot); if (prodRoot1!=prodRoot2) { rootCounter = combineToSameRoot(rootCounter, prodRoot1, prodRoot2, rootsTable); if (isValidToCombine(ringId, consumerId, producerId, graphManager, targetThreadCount)) { int consRoot = rootId(consumerId, rootsTable, lastKnownRoot); int prodRoot = rootId(producerId, rootsTable, lastKnownRoot); rootCounter = combineToSameRoot(rootCounter, consRoot, prodRoot, rootsTable);
buildCountOfStagesForEachThread(graphManager, rootCounter, rootMemberCounter, rootsTable, lastKnownRoot); int root = rootId(stage.stageId, rootsTable, lastKnownRoot); lazyCreateOfArray(rootMemberCounter, stageArrays, root); addsCount = add(stageArrays, stage, root, graphManager, rootsTable, lastKnownRoot, addsCount, int root = rootId(stage.stageId, rootsTable, lastKnownRoot); lazyCreateOfArray(rootMemberCounter, stageArrays, root); addsCount = add(stageArrays, stage, root, graphManager, rootsTable, lastKnownRoot, addsCount, hasNoInternalInputs(graphManager, stage, rootsTable, lastKnownRoot) && int root = rootId(stage.stageId, rootsTable, lastKnownRoot); lazyCreateOfArray(rootMemberCounter, stageArrays, root); addsCount = add(stageArrays, stage, root, graphManager, rootsTable, lastKnownRoot, addsCount, int root = rootId(stage.stageId, rootsTable, lastKnownRoot); lazyCreateOfArray(rootMemberCounter, stageArrays, root); addsCount = add(stageArrays, stage, root, graphManager, rootsTable, lastKnownRoot, addsCount,
final int rootCounter = hierarchicalClassifier(rootsTable, lastKnownRoot, graphManager, targetThreadCount, pipes, countStages, enforceLimit); int id = rootId(stage.stageId, rootsTable, lastKnownRoot); logger.info("finished hierarchical classifications"); PronghornStage[][] stageArrays = buildOrderedArraysOfStages(graphManager, rootCounter, rootsTable, lastKnownRoot); enforceThreadLimit(graphManager, targetThreadCount, stageArrays, comp);
Pipe inPipe = GraphManager.getInputPipe(graphManager, consumerId, i); int inProd = GraphManager.getRingProducerStageId(graphManager, inPipe.id); int tempRootId = rootId(inProd, rootsTable, lastKnownRoot); lazyCreateOfArray(rootMemberCounter, stageArrays, tempRootId); PronghornStage[] prodsList = stageArrays[tempRootId]; int localRootId = rootId(consumerId, rootsTable, lastKnownRoot); if (localRootId==root) { count = add(stageArrays, GraphManager.getStage(graphManager, consumerId), root, graphManager, rootsTable, lastKnownRoot, count, rootMemberCounter, log, false, ensureAllPrevious);
PronghornStage[][] stageArrays = buildStageGroups(graphManager, targetThreadCount, enforceLimit, countStages, rootsTable, lastKnownRoot, comp); verifyNoStagesAreMissing(graphManager, stageArrays);
executorService.execute(buildRunnable(allStagesLatch, ntsArray[i]));
private static StageScheduler defaultSchedulerImpl(GraphManager gm, final boolean threadLimitHard, final int targetThreadCountLimit) { assert(targetThreadCountLimit>0); final int countStages = GraphManager.countStages(gm); //disabled until we find a large machine for testing if (targetThreadCountLimit==Integer.MAX_VALUE) { //NOTE: this case will be rarely used, the other schedules are // more efficient however this scheduler is much simpler. logger.trace("Threads in use {}, one per stage.", countStages); return new ThreadPerStageScheduler(gm); } else { logger.trace("Targeted threads in use {}, fixed limit with fixed script. NOTE: More threads may be used use to graph complexity and telemetry usage.", targetThreadCountLimit); return new ScriptedFixedThreadsScheduler(gm, targetThreadCountLimit, threadLimitHard); } }
PronghornStage[][] orderedStages = ScriptedFixedThreadsScheduler.buildStageGroups(graphManager, 1, true);