/** * * @param gm * @param source _in_ Any input pipe that will be replicated. * @param a _out_ Target pipe; will be joined with b. * @param b _out_ Target pipe; will be joined with a. */ public ReplicatorStage(GraphManager gm, Pipe<T> source, Pipe<T> a, Pipe<T> b) { this(gm,source,join(a,b)); }
public static RawDataJoinerStage newInstance(GraphManager gm, Pipe<RawDataSchema> output, Pipe<RawDataSchema> ... inputs) { return new RawDataJoinerStage(gm, output, inputs); }
@SuppressWarnings("unchecked") public static <T extends MessageSchema<T>> ReplicatorStage<T> newInstance(GraphManager gm, Pipe<T> source, Pipe<T> ... targets) { return new ReplicatorStage<T>(gm,source,targets); }
@Deprecated public static <T extends MessageSchema<T>> ReplicatorStage<T> instance(GraphManager gm, Pipe<T> source, Pipe<T> ... targets) { return newInstance(gm, source, targets); }
findStableCutPoint(ss); if (doneCopy(ss, ss.byteTailPos, ss.source.slabMask & (int)ss.cachedTail, (int)ss.totalPrimaryCopy, ss.totalBytesCopy)) { recordCopyComplete(ss, ss.tempByteTail, ss.totalBytesCopy);
@Override public void shutdown() { //if we are in the middle of a partial copy push the data out, this is blocking while (0!=totalPrimaryCopy) { //if all the copies are done then record it as complete, does as much work as possible each time its called. if (doneCopy(this, byteTailPos, source.slabMask & (int)cachedTail, (int)totalPrimaryCopy, totalBytesCopy)) { recordCopyComplete(this, tempByteTail, totalBytesCopy); } } }
@Override public void run() { processData();//even called when shutting down so we can clear the pipes. processData(); requestShutdown();
public static RawDataSplitterStage newInstance(GraphManager gm, Pipe<RawDataSchema> source, Pipe<RawDataSchema> ... targets) { return new RawDataSplitterStage(gm, source, targets); }
@Override public void run() { processAvailData(this); }
/** * * @param gm * @param input _in_ Schema T input pipes that will be distributed * @param outputs _out_ Pipes on which the input pipe will be distributed on */ public RoundRobinRouteStage(GraphManager gm, Pipe<T> input, Pipe<T> ... outputs) { super(gm,input,outputs); this.inputRing = input; this.outputRings = outputs; this.targetRingInit = outputs.length-1; this.targetRing = targetRingInit; this.supportsBatchedPublish = true; this.supportsBatchedRelease = true; assert(validateTargetSize(input, outputs)) : "output pipes must be as large or larger than input"; }
private static <S extends MessageSchema<S>> boolean doneCopy(ReplicatorStage<S> ss, int byteTailPos, int primaryTailPos, int totalPrimaryCopy, int totalBytesCopy) { int c = 0; int[] working = ss.working; int limit = ss.workingPos; for(int j=0; j<limit; j++) { Pipe<S> target = ss.targets[working[j]]; if (Pipe.hasRoomForWrite(target, totalPrimaryCopy)) { Pipe.confirmLowLevelWriteUnchecked(target, totalPrimaryCopy); copyData(ss, byteTailPos, totalBytesCopy, primaryTailPos, totalPrimaryCopy, target); Pipe.sumWrittenFragments(target, ss.totalFragmentsCopy); Pipe.notifyPubListener(target); } else { working[c++] = working[j]; } } ss.workingPos = c; return 0==c; //returns false when there are still targets to write }
super(gm,source,targets); assert(noDuplicaateTargets(targets)) : "can not replicate to the same target pipe mutiple times";
private static <S extends MessageSchema<S>> void recordCopyComplete(ReplicatorStage<S> ss, int tempByteTail, int totalBytesCopy) { //release tail so data can be written int i = Pipe.BYTES_WRAP_MASK&(tempByteTail + totalBytesCopy); Pipe.markBytesReadBase(ss.source, totalBytesCopy); //record the bytes consumed so far Pipe.setBlobWorkingTail(ss.source, i); Pipe.setBlobTailPosition(ss.source,i); Pipe.publishWorkingTailPosition(ss.source,(ss.cachedTail+=ss.totalPrimaryCopy)); ss.totalPrimaryCopy = 0; //clear so next time we find the next block //both end of pipe was sent AND we have consumed everything off that pipe. if ((0 == (0xF&ss.checkShutdownCycle++)) && Pipe.isEndOfPipe(ss.source, ss.cachedTail) && Pipe.contentRemaining(ss.source)==0) { //we have copied everything including the EOF marker to all the downstream consumers ss.requestShutdown(); } }
this.requestShutdown(); return;
readHeaderForNextBlock(inputStream); targetPipeIdx = -1; readHeaderForNextBlock(inputStream);
Pipe.publishEOF(output); requestShutdown(); } else { int size = Pipe.addMsgIdx(output, RawDataSchema.MSG_CHUNKEDSTREAM_1);
public void buildReplicators(GraphManager gm, ArrayList<ReactiveManagerPipeConsumer> consumers) { int i = inputPipes.length; while (--i>=0) { if (1 == groupedPipes[i].length) { //swap back to using direct connection int c = consumers.size(); while (--c>=0) { if (consumers.get(c).swapIfFound(groupedPipes[i][0], inputPipes[i])) { break; } } if (c<0) { //can not optimize this case so just add the extra hop. logger.info("internal error unable to find this pipe! Hello Guys!"); ReplicatorStage.newInstance(gm, inputPipes[i], groupedPipes[i]); } } else { ReplicatorStage.newInstance(gm, inputPipes[i], groupedPipes[i]); } } }
ReplicatorStage splitter = new ReplicatorStage(gm, getOutputPipe(gm, generator), pipe(busConfig.grow2x()), pipe(busConfig.grow2x())); PronghornStage validateResults = new TestValidator(gm, getOutputPipe(gm, splitter, 2), getOutputPipe(gm, splitter, 1));
public static <T extends MessageSchema<T>> Pipe<T> attach(boolean debug, GraphManager gm, Pipe<T> source, Appendable console) { if (debug) { Pipe<T> out1 = new Pipe<T>(source.config().grow2x()); Pipe<T> out2 = new Pipe<T>(source.config().grow2x()); ReplicatorStage.newInstance(gm, source, out1, out2); ConsoleJSONDumpStage.newInstance(gm, out1, console); return out2; } else { return source; } }
doFinalOutput2); ReplicatorStage.newInstance(gm, encryptedDataPipe, encryptedDataPipeA, encryptedDataPipeB);