/** * Read raw data and route blocks to the right target based on data found. * On the stream there is a packed int for which pipe gets the data and * second packed long for how much data is sent to that location. All outputs * are sent the end of file marker when the end of the file is reached. * It is assumed that each pipe will be consuming multiple chunks of data. * * @param source RawDataSchema Pipe to read from * @param targets an array of outgoing target pipes */ public RawDataSplitterStage(GraphManager gm, Pipe<RawDataSchema> source, Pipe<RawDataSchema> ... targets) { super(gm, source, targets); this.source = source; this.targets = targets; this.inputStream = Pipe.inputStream(source); //TODO: class needs unit test //TODO: class needs to be moved into Pronghorn.. }
/** * Peeks at input stream of specified pipe before Trie parse is run * @param pipe pipe to read * @return stream */ public static <S extends MessageSchema<S>> DataInputBlobReader<S> peekInputStream(Pipe<S> pipe, int loc) { assert(LOCUtil.isLocOfAnyType(loc, TypeMask.TextASCII, TypeMask.TextASCIIOptional, TypeMask.TextUTF8, TypeMask.TextUTF8Optional, TypeMask.ByteVector, TypeMask.ByteVectorOptional)): "Value found "+LOCUtil.typeAsString(loc)+" b"+Integer.toBinaryString(loc); DataInputBlobReader<S> stream = Pipe.inputStream(pipe); DataInputBlobReader.peekHighLevelAPIField(stream, loc); return stream; }
public static <S extends MessageSchema<S>> DataInputBlobReader<S> inputStream(Pipe<S> pipe, int loc) { assert(LOCUtil.isLocOfAnyType(loc, TypeMask.TextASCII, TypeMask.TextASCIIOptional, TypeMask.TextUTF8, TypeMask.TextUTF8Optional, TypeMask.ByteVector, TypeMask.ByteVectorOptional)): "Value found "+LOCUtil.typeAsString(loc)+" b"+Integer.toBinaryString(loc); DataInputBlobReader<S> stream = Pipe.inputStream(pipe); stream.openHighLevelAPIField(loc); return stream; }
public static <S extends MessageSchema<S>> DataInputBlobReader<S> inputStream(Pipe<S> pipe, int loc) { assert(LOCUtil.isLocOfAnyType(loc, TypeMask.TextASCII, TypeMask.TextASCIIOptional, TypeMask.TextUTF8, TypeMask.TextUTF8Optional, TypeMask.ByteVector, TypeMask.ByteVectorOptional)): "Value found "+LOCUtil.typeAsString(loc)+" b"+Integer.toBinaryString(loc); DataInputBlobReader<S> stream = Pipe.inputStream(pipe); stream.openHighLevelAPIField(loc); return stream; }
/** * Peeks at input stream of specified pipe before Trie parse is run * @param pipe pipe to read * @return stream */ public static <S extends MessageSchema<S>> DataInputBlobReader<S> peekInputStream(Pipe<S> pipe, int loc) { assert(LOCUtil.isLocOfAnyType(loc, TypeMask.TextASCII, TypeMask.TextASCIIOptional, TypeMask.TextUTF8, TypeMask.TextUTF8Optional, TypeMask.ByteVector, TypeMask.ByteVectorOptional)): "Value found "+LOCUtil.typeAsString(loc)+" b"+Integer.toBinaryString(loc); DataInputBlobReader<S> stream = Pipe.inputStream(pipe); DataInputBlobReader.peekHighLevelAPIField(stream, loc); return stream; }
/** * Peeks at input stream of specified pipe before Trie parse is run * @param pipe pipe to read * @return stream */ public static <S extends MessageSchema<S>> DataInputBlobReader<S> peekInputStream(Pipe<S> pipe, int loc) { assert(LOCUtil.isLocOfAnyType(loc, TypeMask.TextASCII, TypeMask.TextASCIIOptional, TypeMask.TextUTF8, TypeMask.TextUTF8Optional, TypeMask.ByteVector, TypeMask.ByteVectorOptional)): "Value found "+LOCUtil.typeAsString(loc)+" b"+Integer.toBinaryString(loc); DataInputBlobReader<S> stream = Pipe.inputStream(pipe); DataInputBlobReader.peekHighLevelAPIField(stream, loc); return stream; }
public static <S extends MessageSchema<S>> DataInputBlobReader<S> inputStream(Pipe<S> pipe, int loc) { assert(LOCUtil.isLocOfAnyType(loc, TypeMask.TextASCII, TypeMask.TextASCIIOptional, TypeMask.TextUTF8, TypeMask.TextUTF8Optional, TypeMask.ByteVector, TypeMask.ByteVectorOptional)): "Value found "+LOCUtil.typeAsString(loc)+" b"+Integer.toBinaryString(loc); DataInputBlobReader<S> stream = Pipe.inputStream(pipe); stream.openHighLevelAPIField(loc); return stream; }
this.sourceBuffer = Pipe.blob(input); this.sourceMask = Pipe.blobMask(input); this.inputStream = Pipe.inputStream(input);
public static boolean accumulateInputStream(Pipe<RawDataSchema> pipe) { int msgIdx = Pipe.takeMsgIdx(pipe); boolean isEnd; if (RawDataSchema.MSG_CHUNKEDSTREAM_1 == msgIdx) { isEnd = DataInputBlobReader.accumLowLevelAPIField(Pipe.inputStream(pipe))<=0; Pipe.confirmLowLevelRead(pipe, Pipe.sizeOf(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1)); } else { //eof isEnd = true; Pipe.confirmLowLevelRead(pipe, Pipe.EOF_SIZE); } Pipe.readNextWithoutReleasingReadLock(pipe); return isEnd; }
public static boolean accumulateInputStream(Pipe<RawDataSchema> pipe) { int msgIdx = Pipe.takeMsgIdx(pipe); boolean isEnd; if (RawDataSchema.MSG_CHUNKEDSTREAM_1 == msgIdx) { isEnd = DataInputBlobReader.accumLowLevelAPIField(Pipe.inputStream(pipe))<=0; Pipe.confirmLowLevelRead(pipe, Pipe.sizeOf(pipe, RawDataSchema.MSG_CHUNKEDSTREAM_1)); } else { //eof isEnd = true; Pipe.confirmLowLevelRead(pipe, Pipe.EOF_SIZE); } Pipe.readNextWithoutReleasingReadLock(pipe); return isEnd; }
ChannelReader reader = Pipe.inputStream(pipe); int startingAvailable = reader.available(); if (isEnd && startingAvailable==0) {
ChannelReader reader = Pipe.inputStream(pipe); int startingAvailable = reader.available(); if (isEnd && startingAvailable==0) {
private void processData() { DataInputBlobReader<RawDataSchema> inputStream = Pipe.inputStream(source);
case RawDataSchema.MSG_CHUNKEDSTREAM_1: DataInputBlobReader<RawDataSchema> fieldByteArray = Pipe.inputStream(input); int additionalLength = fieldByteArray.accumLowLevelAPIField();
@Override public void startup() { //load log file and parse out all the telemetry snapshots... lineParser = new TrieParser(); lineParser.setUTF8Value("greenlightning: digraph gl {\n", 1); lineParser.setUTF8Value("digraph gl {\n", 1); lineParser.setUTF8Value("greenlightning: }\n", 2); lineParser.setUTF8Value("}\n", 2); lineParser.setUTF8Value("greenlightning: %b\n", 3); lineParser.setUTF8Value("%b\n", 3); reader = TrieParserReaderLocal.get(); logFileInput = Pipe.inputStream(logFile); //start // [8:RProxy] INFO com.ociweb.pronghorn.stage.scheduling.GraphManager - // 20181212215745711.dot // digraph gl { //body //stop // } }
DataInputBlobReader<RawDataSchema> localReader = Pipe.inputStream(buffer); localReader.openLowLevelAPIField();
case RawDataSchema.MSG_CHUNKEDSTREAM_1: DataInputBlobReader<RawDataSchema> reader = Pipe.inputStream(input); int payloadLen = Pipe.peekInt(input, 1); //length is after meta data
int j = inputs.length; while (--j>=0) { total += Pipe.inputStream(inputs[j]).readPackedLong();
DataInputBlobReader<?> inputStream = Pipe.inputStream(request);
if (Pipe.inputStream(source).available()>0) {