public MsgRuntime(String[] args, String name) { this.gm = new GraphManager(name); this.args = args != null ? args : new String[0]; }
/** * Constructor for log-telemetry * @param inputFilePath The input path for the file to be logged * @param port The port on which telemetry will run * @param out An appendable in which the specified file gets written to */ public LogTelemetry (String logFilePath) { // Add edges and pipes populateGraph(gm, "127.0.0.1", 8099, logFilePath); // Turning on the telemetry web page is as simple as adding this line // It will be accessible on the most public IP it can find by default gm.enableTelemetry("127.0.0.1",8098); }
private static PronghornStage findStageByPath(GraphManager m, PronghornStage stage, int idx, int[] path) { if (idx>=path.length) { return stage; } return findStageByPath(m,getRingConsumer(m, getOutputPipe(m,stage,path[idx]).id),1+idx,path); }
public static void recordThreadGroup(PronghornStage stage, final int group, final GraphManager graphManager) { //logger.info("recorded group for {} {}",group,stage); addNota(graphManager, THREAD_GROUP, group, stage); buldStageDOTName(graphManager, stage); }
private static int dotEstimate(GraphManager graphManager) { return (300*GraphManager.countStages(graphManager))+ (400*GraphManager.allPipes(graphManager).length); }
int stages = GraphManager.countStages(m); if (mustShowOnTelemetry(m, stage)) { populateRanks(m, ranks, stage, stageId); target.write(stageDisplayName); } else { stageDisplayName = buldStageDOTName(m, stage); target.write(stageDisplayName); writeElapsed(target, ElapsedTimeRecorder.elapsedAtPercentile(m.stageElapsed[stage.stageId],(float)percentile)); Object background = getNota(m, stage.stageId, GraphManager.DOT_BACKGROUND, "gray90"); if (null!=background) { target.append(",style=filled,fillcolor="); int producer = getRingProducerId(m, j); int consumer = getRingConsumerId(m, j); ((!(GraphManager.getStage(m, consumer) instanceof PipeMonitorCollectorStage)) && (!(stageForMonitorData(m,GraphManager.getStage(m, consumer))))) ) { final int outputPipeCount = GraphManager.getOutputPipeCount(m, producer); int sum=0; int lastPipeId=-1; for(int p=1;p<=outputPipeCount;p++) {
@Test public void constructionOfSimpleGraph() { GraphManager gm = new GraphManager(); assertTrue(a == GraphManager.getStage(gm,a.stageId)); assertTrue(b == GraphManager.getStage(gm,b.stageId)); assertTrue(c == GraphManager.getStage(gm,c.stageId)); assertTrue(rb1 == GraphManager.getPipe(gm,rb1.id)); assertTrue(rb2 == GraphManager.getPipe(gm,rb2.id)); assertTrue(a == GraphManager.getRingProducer(gm,rb1.id)); assertTrue(b == GraphManager.getRingConsumer(gm,rb1.id)); assertTrue(b == GraphManager.getRingProducer(gm,rb2.id)); assertTrue(c == GraphManager.getRingConsumer(gm,rb2.id)); assertTrue(GraphManager.mayHaveUpstreamData(gm, c.stageId)); //this is true because the first ring buffer has 1 integer GraphManager.setStateToStopping(gm, a.stageId); assertTrue(GraphManager.mayHaveUpstreamData(gm, c.stageId)); //this is true because the first ring buffer has 1 integer GraphManager.setStateToStopping(gm, a.stageId); GraphManager.setStateToStopping(gm, b.stageId); assertTrue(GraphManager.mayHaveUpstreamData(gm, c.stageId)); //this is true because the first ring buffer has 1 integer
private void pipelineTest(boolean highLevelAPI, boolean monitor, boolean useTap, boolean useRouter) { GraphManager gm = new GraphManager(); GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, Integer.valueOf(0), productionStage); GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, Integer.valueOf(0), highLevelAPI ? GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, Integer.valueOf(0), new RoundRobinRouteStage(gm, rings[j++], splitsBuffers)); } else { GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, Integer.valueOf(0), new ReplicatorStage(gm, rings[j++], splitsBuffers)); GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, Integer.valueOf(0), highLevelAPI ? new CopyStageHighLevel(gm, outputRing1, inputRing) : new CopyStageLowLevel(gm, outputRing1, inputRing)); boolean useRoute = useTap&useRouter; PronghornStage dumpStage = highLevelAPI ? new DumpStageHighLevel(gm, inputRing, useRoute) : new DumpStageLowLevel(gm, inputRing, useRoute); GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, Integer.valueOf(0), dumpStage); GraphManager.enableBatching(gm); ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(GraphManager.cloneAll(gm)); scheduler.playNice = true; scheduler.startup(); GraphManager.blockUntilStageBeginsShutdown(gm, productionStage, TIMEOUT_SECONDS*1000); System.err.println(GraphManager.getRingProducer(gm, rings[k].id)+" ->\n "+rings[k].toString()+" -> "+GraphManager.getRingConsumer(gm, rings[k].id)); k++; System.err.println(GraphManager.getRingConsumer(gm, rings[k-1].id));
private void scanForPipesWithData(GraphManager g, String label) { Pipe<?>[] allPipes = GraphManager.allPipes(g); int a = allPipes.length; while (--a >= 0) { Pipe p = allPipes[a]; if (null != p) { PronghornStage stagePro = GraphManager.getRingProducer(g, p.id); PronghornStage stageCon = GraphManager.getRingConsumer(g, p.id); if (!GraphManager.hasNota(g, stagePro.stageId, GraphManager.MONITOR)) { if (!Pipe.isEmpty(p)) { System.out.println(label+p+" "+stagePro+"->"+stageCon); } }; } } }
public static void main(String[] args) { String home = System.getenv().get("HOME"); String filesPath = MainArgs.getOptArg("files", "-f", args, (null==home?"~":home)+"/www"); String host = MainArgs.getOptArg("host", "-h", args, null); int port = Integer.valueOf(MainArgs.getOptArg("port", "-p", args, "8080")); GraphManager gm = new GraphManager(); populateGraph(gm, host, port, filesPath); gm.enableTelemetry(8089); StageScheduler.defaultScheduler(gm).startup(); }
@Test public void notaTests() { final GraphManager gm = new GraphManager(); GraphManager.addNota(gm, GraphManager.UNSCHEDULED, GraphManager.UNSCHEDULED, a); PronghornStage b = new SimpleInOut(gm,rb1,rb2); GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, 20000, b); PronghornStage c = new SimpleIn(gm,rb2); GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, 10000, c); assertEquals(a, GraphManager.getStageWithNotaKey(gm, GraphManager.UNSCHEDULED, 1)); GraphManager.getStageWithNotaKey(gm, GraphManager.UNSCHEDULED, 2); fail("should not have found"); } catch (UnsupportedOperationException uoe) { assertEquals(2, GraphManager.countStagesWithNotaKey(gm, GraphManager.SCHEDULE_RATE)); assertEquals(1, GraphManager.countStagesWithNotaKey(GraphManager.cloneStagesWithNotaKeyValue(gm, GraphManager.SCHEDULE_RATE, 20000), GraphManager.SCHEDULE_RATE)); assertEquals(2, GraphManager.countStagesWithNotaKey(GraphManager.cloneStagesWithNotaKey(gm, GraphManager.SCHEDULE_RATE), GraphManager.SCHEDULE_RATE)); GraphManager.addNota(gm, GraphManager.SCHEDULE_RATE, 30000, b); //change value assertEquals(0, GraphManager.countStagesWithNotaKey(GraphManager.cloneStagesWithNotaKeyValue(gm, GraphManager.SCHEDULE_RATE, 20000), GraphManager.SCHEDULE_RATE)); GraphManager.addNota(gm, Integer.toHexString(j), j, a); GraphManager.addNota(gm, "ANota", 70000, a, b, c); assertEquals(3, GraphManager.countStagesWithNotaKey(gm, "ANota"));
int inputs = GraphManager.getInputPipeCount(graphManager, stage.stageId); int inputPipeId = GraphManager.getInputPipeId(graphManager, stage.stageId, j); PronghornStage prod = GraphManager.getRingProducer(graphManager, inputPipeId); int outputCount = GraphManager.getOutputPipeCount(graphManager, stage.stageId); boolean simpleAddInOrder = GraphManager.hasNota(graphManager, stage.stageId, GraphManager.LOAD_BALANCER); Pipe outputPipe = GraphManager.getOutputPipe(graphManager, stage, r); count = recursiveAdd(stageArrays, root, graphManager, rootsTable, lastKnownRoot, count, rootMemberCounter, log, GraphManager.getRingConsumerId(graphManager, outputPipe.id),ensureAllPrevious); int consumerId = GraphManager.getRingConsumerId( graphManager, GraphManager.getOutputPipe( graphManager, stage, if (GraphManager.hasNota(graphManager, consumerId, GraphManager.TRIGGER)) { tempRanks[r] = -1; } else { rootsTable, lastKnownRoot, count, rootMemberCounter, log, GraphManager.getRingConsumerId(graphManager, GraphManager.getOutputPipe(graphManager, stage, r).id), ensureAllPrevious);
stages = GraphManager.countStages(graphManager); while (stages>=0) { PronghornStage stage = GraphManager.getStage(graphManager, stages--); if (null!=stage) { int inputCount = GraphManager.getInputPipeCount(graphManager, stage); lazyCreateOfArray(rootMemberCounter, stageArrays, root); int ringProducerStageId = GraphManager.getRingProducerStageId(graphManager, GraphManager.getInputPipe(graphManager, stage, j).id); if (rootId(ringProducerStageId, rootsTable) != root || GraphManager.isStageInLoop(graphManager, stage.stageId)>=-1 ) { isTop = true; if (0==inputCount || null!=GraphManager.getNota(graphManager, stage.stageId, GraphManager.PRODUCER, null)) { isTop=true;
public static GraphManager cloneStagesWithNotaKey(GraphManager m, Object key) { GraphManager clone = new GraphManager( new GraphManagerStageStateData(), m.recordTypeData, m.name); //register each stage int i = m.stageIdToStage.length; while (--i>=0) { PronghornStage stage = m.stageIdToStage[i]; if (null!=stage) { //copy this stage if it has the required key if (m != getNota(m, stage, key, m)) { copyStage(m, clone, stage); copyNotasForStage(m, clone, stage); } } } return clone; }
GraphManager gm = new GraphManager(); gm.enableTelemetry(8089); scheduler.startup(); while (!GraphManager.isStageTerminated(gm, watch.stageId) ) { scheduler.run(); Thread.yield();
PronghornStage[] neverStarted = GraphManager.allStagesByState(m, GraphManagerStageStateData.STAGE_NEW); int j = neverStarted.length; if (j>0) { PronghornStage[] neverShutdown = GraphManager.allStagesByState(m, GraphManagerStageStateData.STAGE_STARTED); j = neverShutdown.length; if (j>0) { PronghornStage[] neverTerminated = GraphManager.allStagesByState(m, GraphManagerStageStateData.STAGE_STOPPING); j = neverTerminated.length; if (j>0) { while (++i<m.stageIdToStage.length) { if (null!=m.stageIdToStage[i]) { if (!isStageTerminated(m, i) ) { PronghornStage stage = getStage(m,i); if (!isInputsEmpty(m,stage)) { logInputs(StageScheduler.logger, m, stage); StageScheduler.logger.error(" Expected stage {} to be stopped but it appears to be running. terminated:{}", stage, isStageTerminated(m, i)); logOutputs(StageScheduler.logger, m, stage); StageScheduler.logger.error("-------------------");//divide the log for better clarity reportUnexpectedThreadStacks(m);
protected PronghornStage(GraphManager graphManager, Pipe[] inputs, Pipe[] outputs) { assert(null!=inputs) : "Use NONE"; assert(null!=outputs) : "Use NONE"; //logger.trace("register new pronghorn stage started"); this.stageId = GraphManager.newStageId(graphManager); this.boxedStageId = this.stageId; this.hash = PronghornStage.class.hashCode() ^ stageId; this.graphManager = graphManager; GraphManager.register(graphManager, this, inputs, outputs); GraphManager.addNota(graphManager, GraphManager.THREAD_GROUP, null, this); //This provides room for assignment later //logger.trace("register new pronghorn stage done"); }
public static File writeTapeToFileUsingPipe(Pipe<RawDataSchema> inputPipe, byte[] testData) throws IOException, FileNotFoundException { GraphManager gm = new GraphManager(); File f2 = File.createTempFile("fileTapeWriteTest", "dat"); f2.deleteOnExit(); new ByteArrayProducerStage(gm, testData, inputPipe); new TapeWriteStage(gm, inputPipe, new RandomAccessFile(f2,"rw")); GraphManager.enableBatching(gm); ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm); scheduler.startup(); scheduler.awaitTermination(3, TimeUnit.SECONDS); return f2; }
private int[] buildCountOfStagesForEachThread(GraphManager graphManager, int rootCounter, int[] rootMemberCounter) { //long start = System.nanoTime(); Arrays.fill(rootMemberCounter, 0); int countStages = GraphManager.countStages(graphManager); for(int stages=1; stages <= countStages; stages++) { PronghornStage stage = GraphManager.getStage(graphManager, stages); if (null!=stage) { int rootId = rootId(stage.stageId, rootsTable); rootMemberCounter[rootId]++; GraphManager.addNota(graphManager, GraphManager.THREAD_GROUP, rootId, stage); } } //System.err.println("performance issue here calling this too often "+countStages+" duration "+(System.nanoTime()-start)); //logger.info("group counts "+Arrays.toString(rootMemberCounter)); return rootMemberCounter; }