private static void speedTestEncodeDecodeParallel() { GraphManager gm = new GraphManager(); PipeConfig phastCodeConfig = new PipeConfig(PhastCodecSchema.instance, 256); //TODO: AA, must optimize queue size based on load. Build tool for this. PipeConfig phastCodeConfig2 = new PipeConfig(RawDataSchema.instance, 10,1000); //TODO: AA, must optimize queue size based on load. Build tool for this. PipeConfig rawDataConfig = new PipeConfig(RawDataSchema.instance, 4,10*63*1); Pipe inputPipe1 = new Pipe(phastCodeConfig); Pipe inputPipe1B = new Pipe(phastCodeConfig2); Pipe inputPipe2 = new Pipe(phastCodeConfig); Pipe inputPipe2B = new Pipe(phastCodeConfig2); // Pipe inputPipe3 = new Pipe(phastCodeConfig); Pipe packedDataPipe1 = new Pipe(rawDataConfig); Pipe packedDataPipe2 = new Pipe(rawDataConfig); // Pipe packedDataPipe3 = new Pipe(rawDataConfig); Pipe packedDataPipeFinal = new Pipe(rawDataConfig); Pipe outputPipe = new Pipe(phastCodeConfig); Pipe rePackedDataPipe = new Pipe(rawDataConfig); //Add production stage? int iterations = 3000000;//*1000; LongDataGenStage genStage = new LongDataGenStage(gm, new Pipe[]{inputPipe1, inputPipe2}, iterations); PhastPackingStage encodeStage1 = new PhastPackingStage(gm, inputPipe1, inputPipe1B, packedDataPipe1); PhastPackingStage encodeStage2 = new PhastPackingStage(gm, inputPipe2, inputPipe2B, packedDataPipe2); // PhastEncodeStage encodeStage3 = new PhastEncodeStage(gm, inputPipe3, packedDataPipe3, chunkSize); // PhastDecodeStage decodeStage = new PhastDecodeStage(gm, packedDataPipe, outputPipe ); // PhastEncodeStage encodeStage2 = new PhastEncodeStage(gm, outputPipe, rePackedDataPipe ); // PipeCleanerStage dumpStage = new PipeCleanerStage(gm, rePackedDataPipe); // PipeCleanerStage dumpStage = new PipeCleanerStage(gm, outputPipe); MergeRawDataSchemaStage mergeStage = new MergeRawDataSchemaStage(gm, new Pipe[]{packedDataPipe1, packedDataPipe2},packedDataPipeFinal); // PhastDecodeStage decodeStage = new PhastDecodeStage(gm, packedDataPipeFinal, outputPipe ); PipeCleanerStage dumpStage0 = new PipeCleanerStage(gm, packedDataPipeFinal); // PipeCleanerStage dumpStage1 = new PipeCleanerStage(gm, packedDataPipe1); // PipeCleanerStage dumpStage2 = new PipeCleanerStage(gm, packedDataPipe2); // PipeCleanerStage dumpStage3 = new PipeCleanerStage(gm, packedDataPipe3); // PipeCleanerStage dumpStage = new PipeCleanerStage(gm, outputPipe); // GraphManager.enableBatching(gm); //due to internal batching nature of stages this does not help PipeMonitorCollectorStage monitor = PipeMonitorCollectorStage.attach(gm);//TODO: only gets triggered on shutdown call, TODO: need to fix this. final ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm); scheduler.playNice = false; //this may or may not help scheduler.startup(); //This thread must park a little or the monitor does not collect any results, NOTE: needs investigation why this is true, almost spooky action at a distance... LockSupport.parkUntil(System.currentTimeMillis()+500); scheduler.awaitTermination(500, TimeUnit.SECONDS);//blocks until genStage requests shutdown or timeout }

How to useMergeRawDataSchemaStage in com.ociweb.pronghorn.stage.phast

Best Java code snippets using com.ociweb.pronghorn.stage.phast.MergeRawDataSchemaStage (Showing top 2 results out of 315)

How to use
MergeRawDataSchemaStage
in
com.ociweb.pronghorn.stage.phast