protected PhastPackingStage(GraphManager graphManager, Pipe<PhastCodecSchema> input1, Pipe<RawDataSchema> input2, Pipe<RawDataSchema> output) { super(graphManager, join(input1,input2), output); this.input1 = input1; this.input2 = input2; this.output = output; //Does not need batching to be done by producer or consumer this.supportsBatchedPublish = false; this.supportsBatchedRelease = false; }
protected PhastUnpackingStage(GraphManager graphManager, Pipe<RawDataSchema> input, Pipe<PhastCodecSchema> output1, Pipe<RawDataSchema> output2) { super(graphManager, input, join(output1, output2)); this.input = input; this.output1 = output1; this.output2 = output2; }
@Override public void run() { pump(input1, input2, output, lengthLookup, writer, input2Reader); }
switch(msgIdx) { case MSG_BLOBCHUNK_1000: consumeBlobChunk(input); break; case MSG_001_10001: consume001(input); break; case MSG_002_10002: consume002(input); break; case MSG_003_10003: consume003(input); break; case MSG_004_10004: consume004(input); break; case MSG_005_10005: consume005(input); break; case MSG_006_10006: consume006(input); break; case MSG_007_10007: consume007(input); break; case MSG_008_10008: consume008(input); break; case MSG_009_10009:
long test = PhastDecoder.decodePresentLong(reader, 0, 0, false); assertTrue(test==3894); test = PhastDecoder.decodeDeltaLong(longDictionary, reader, 0, 1, 0, false); long test2 = PhastDecoder.decodeDeltaLong(longDictionary, reader, 1, 1, 1, false); assertTrue(test==961 && test2==961); test = PhastDecoder.decodeIncrementLong(longDictionary, 0, 0, 0, false); test2 = PhastDecoder.decodeIncrementLong(longDictionary, 1, 0, 1, false); assertTrue(test == 4 && test2 == 4); test = PhastDecoder.decodeCopyLong(longDictionary, reader, 0, 2, 0, false); test2 = PhastDecoder.decodeCopyLong(longDictionary, reader, 1, 2, 1, false); assertTrue(test == 70 && test2 == 903); test = PhastDecoder.decodeDefaultLong(reader, 0, defaultValues, 0, 2, false); test2 = PhastDecoder.decodeDefaultLong(reader, 1, defaultValues, 1, 2, false); assertTrue(test == 16 && test2==404);
PhastEncoder.encodeLongPresent(writer, 0, 1, defaultTest, false); PhastEncoder.incrementLong(longDictionary, writer, 1, 1, 4, false); PhastEncoder.incrementLong(longDictionary, writer, 0, 1, 4, false); PhastEncoder.copyLong(longDictionary, writer, 0, 1, 4, 0, false); PhastEncoder.encodeDefaultLong(defaultLongDictionary, writer, 0, 1, 2, defaultTest, false); PhastEncoder.encodeDefaultLong(defaultLongDictionary, writer, 1, 1, 2, defaultTest, false);
PhastEncoder.encodeShortPresent(writer, 0, 1, defaultTest, false); PhastEncoder.incrementShort(shortDictionary, writer, 1, 1, 4, false); PhastEncoder.incrementShort(shortDictionary, writer, 0, 1, 4, false); PhastEncoder.copyShort(shortDictionary, writer, 0, 1, 4, (short)0, false); PhastEncoder.encodeDefaultShort(defaultShortDictionary, writer, 0, 1, 2, defaultTest, false); PhastEncoder.encodeDefaultShort(defaultShortDictionary, writer, 1, 1, 2, defaultTest, false);
writeBytesToOutput(output1, output2, output2Writer, reader); } else { return;//try later and read bytesRemainingToCopy requestShutdown(); break; closePackedFieldsMessage(msgIdxPos, idxReverseLookup[localFieldCount], output1); localFieldCount = 0; writeBytesToOutput(output1, output2, output2Writer, reader); } else { return;//try later and read bytesRemainingToCopy closePackedFieldsMessage(msgIdxPos, idxReverseLookup[localFieldCount], output1); localFieldCount = 0; closePackedFieldsMessage(msgIdxPos, idxReverseLookup[localFieldCount], output1); localFieldCount = 0;
LongDataGenStage genStage = new LongDataGenStage(gm, new Pipe[]{inputPipe1, inputPipe2}, iterations); PhastPackingStage encodeStage1 = new PhastPackingStage(gm, inputPipe1, inputPipe1B, packedDataPipe1); PhastPackingStage encodeStage2 = new PhastPackingStage(gm, inputPipe2, inputPipe2B, packedDataPipe2); MergeRawDataSchemaStage mergeStage = new MergeRawDataSchemaStage(gm, new Pipe[]{packedDataPipe1, packedDataPipe2},packedDataPipeFinal);
LongDataGenStage genStage = new LongDataGenStage(gm, new Pipe[]{inputPipe}, iterations); PhastPackingStage packStage = new PhastPackingStage(gm, inputPipe, inputPipe2, packedDataPipe); PhastUnpackingStage unPackStage = new PhastUnpackingStage(gm, packedDataPipe, outputPipe1, outputPipe2 ); PipeCleanerStage<PhastCodecSchema> dumpStage1 = new PipeCleanerStage<PhastCodecSchema>(gm, outputPipe1); PipeCleanerStage<RawDataSchema> dumpStage2 = new PipeCleanerStage<RawDataSchema>(gm, outputPipe2);
private void combineContentForSingleMessage(Pipe<PhastCodecSchema> localInput1, Pipe<RawDataSchema> localInput2, Pipe<RawDataSchema> localOutput, short[] lookup, DataOutputBlobWriter<RawDataSchema> localWriter, int size, int maxPerMsg, int outputMaxLen, boolean holdingReleaseReadLock, DataOutputBlobWriter<RawDataSchema> writer) { int localSum = 0; int avail = 0; while ( ( (avail = (outputMaxLen - writer.length())) >= maxPerMsg) && Pipe.hasContentToRead(localInput1)) { if (holdingReleaseReadLock) { Pipe.releaseReadLock(localInput1); holdingReleaseReadLock = false; } int msgIdx = Pipe.takeMsgIdx(localInput1); if (PhastCodecSchema.MSG_BLOBCHUNK_1000 != msgIdx) { localSum = writePackedFields(localSum, localInput1, localWriter, msgIdx, (int) lookup[msgIdx]); } else { localSum = writeByteData(localSum, localInput2, avail); } Pipe.confirmLowLevelRead(localInput1, Pipe.sizeOf(localInput1, msgIdx)); holdingReleaseReadLock = true; } assert(holdingReleaseReadLock); totalLongs += localSum; }
public static void main(String[] args) { speedTestPackUnpacke(); //speedTestEncodeDecodeParallel(); }
private int writePackedFields(int localSum, Pipe<PhastCodecSchema> localInput1, DataOutputBlobWriter<RawDataSchema> localWriter, int msgIdx, int localFieldCount) { int i = localFieldCount; //System.err.println("field count "+i); while (--i >= 0) { packField(localWriter, Pipe.takeLong(localInput1)); } localSum += localFieldCount; return localSum; }
@Test public void copyIntTest() throws IOException{ //create blob for test Pipe<RawDataSchema> encodedValuesToValidate = new Pipe<RawDataSchema>(new PipeConfig<RawDataSchema>(RawDataSchema.instance, 100, 4000)); encodedValuesToValidate.initBuffers(); DataOutputBlobWriter<RawDataSchema> writer = new DataOutputBlobWriter<RawDataSchema>(encodedValuesToValidate); //create int dictionary int[] intDictionary = new int[5]; Arrays.fill(intDictionary, 0); intDictionary[2] = 5; //make it increment 2 values 0 and 5 PhastEncoder.copyInt(intDictionary, writer, 0, 0, 2, 0, false); writer.close(); }
@Test public void defaultIntTest() throws IOException{ //create a blob to test Pipe<RawDataSchema> encodedValuesToValidate = new Pipe<RawDataSchema>(new PipeConfig<RawDataSchema>(RawDataSchema.instance, 100, 4000)); encodedValuesToValidate.initBuffers(); DataOutputBlobWriter<RawDataSchema> writer = new DataOutputBlobWriter<RawDataSchema>(encodedValuesToValidate); //make int array int[] defaultInt = new int[5]; defaultInt[3] = 4; //should encode 16 PhastEncoder.encodeDefaultInt(defaultInt, writer, 1, 1, 3, 16, false); //should encode 4 PhastEncoder.encodeDefaultInt(defaultInt, writer, 0, 1, 3, 16, false); writer.close(); DataInputBlobReader<RawDataSchema> reader = new DataInputBlobReader<RawDataSchema>(encodedValuesToValidate); int test1 = reader.readPackedInt(); //shouldnt encode anything reader.close(); }
combineContentForSingleMessage( localInput1, localInput2, localOutput, lookup, localWriter, size, maxBytesPerMessage, localOutput.maxVarLen,
@Test public void testEncodeString() throws IOException{ //create a new blob pipe to put a string on Pipe<RawDataSchema> pipe = new Pipe<RawDataSchema>(new PipeConfig<RawDataSchema>(RawDataSchema.instance, 100, 4000)); pipe.initBuffers(); DataOutputBlobWriter<RawDataSchema> writer = new DataOutputBlobWriter<RawDataSchema>(pipe); //encode a string on blob using the static method StringBuilder testString = new StringBuilder("This is a test"); PhastEncoder.encodeString(writer, testString , 0, 0, false); writer.close(); //check what is on the pipe DataInputBlobReader<RawDataSchema> reader = new DataInputBlobReader<RawDataSchema>(pipe); //should be -63 int test = reader.readPackedInt(); //the string String value = reader.readUTF(); reader.close(); String s = value.toString(); assertTrue((test==-63) && (s.compareTo("This is a test")==0)); }
@Test public void decodeStringTest() throws IOException{ Pipe<RawDataSchema> pipe = new Pipe<RawDataSchema>(new PipeConfig<RawDataSchema>(RawDataSchema.instance, 100, 4000)); pipe.initBuffers(); DataOutputBlobWriter<RawDataSchema> writer = new DataOutputBlobWriter<RawDataSchema>(pipe); DataOutputBlobWriter.writePackedInt(writer, -63); writer.writeUTF("This is a test"); writer.close(); DataInputBlobReader<RawDataSchema> reader = new DataInputBlobReader<RawDataSchema>(pipe); String stest = PhastDecoder.decodeString(reader, false); reader.close(); assertTrue(stest.compareTo("This is a test") == 0); }