/** * * @param idx int arg index number * @param target * @param <A> * @return length */ public <A extends Appendable> A copyExtractedUTF8ToAppendable(int idx, A target) { return TrieParserReader.capturedFieldBytesAsUTF8(tpr, idx, target); }
public static long parseNext(TrieParserReader reader, TrieParser trie) { return parseNext(reader,trie,-1,-1); }
public static <A extends Appendable> void parse(Pipe<?> pipe, TrieParserReader reader, JSONVisitor visitor) { TrieParserReader.parseSetup(reader, pipe); do { parseValueToken(reader, visitor); } while (TrieParserReader.parseHasContent(reader)); }
private static boolean validateParseEnd(HTTP1xRouterStage that, final int idx) { if (0==TrieParserReader.parseHasContentLength(that.trieReader) || 0==that.inputLengths[idx]) {//(that.inputLengths[idx]-len2)) { //the last byte we jsut parsed if everything is empty must be the \n marker that.trieReader.moveBack(1); return 10==that.trieReader.parseSkipOne(); } return true; }
private static long fieldIdLookup(TrieParser fieldParser, byte[] value, int valuePos, int valueLen, int mask) { //we are trusting escape analysis to not create GC here. return TrieParserReader.query(new TrieParserReader(true), fieldParser, value, valuePos, valueLen, mask); }
public static long convertTextToLong(TrieParserReader parserReader, byte[] backing, int position, int mask, int available) { return (TrieParserReader.query(parserReader, textToNumberTrieParser(), backing, position, available, mask)>=0) ? TrieParserReader.capturedLongField(parserReader, 0) : -1; }
public void store(byte[] backing, int pos, int len, int mask) { if (TrieParserReader.query(reader, DataInputBlobReader.textToNumberTrieParser(), backing, pos, len, mask)>=0) { this.e = TrieParserReader.capturedDecimalEField(reader,0); this.m = TrieParserReader.capturedDecimalMField(reader,0); } else { //TODO: We should use the Rabin Fingerprint to have a strong model of collision this.e = 0; this.m = MurmurHash.hash32(backing, pos, len, 314-579-0066); } }
TrieParserReader reader = new TrieParserReader(true); Process process = Runtime.getRuntime().exec("jstack -l "+getPid()); InputStream stream = process.getInputStream(); TrieParserReader.parseSetup(reader, buffer, 0, bufferPos, Integer.MAX_VALUE); while (TrieParserReader.parseHasContent(reader)) { long id = reader.parseNext(parser); TrieParserReader.capturedFieldBytesAsUTF8(reader, 0, System.err); return false; TrieParserReader.debugAsUTF8(reader, System.err); return false; reader.capturedFieldBytesAsUTF8(reader, 0, System.err); System.err.println();
DataInputBlobReader<RawDataSchema> stream = Pipe.openInputStream(input); TrieParserReader.parseSetup(reader, stream); while (TrieParserReader.parseHasContent(reader)) { long id = reader.parseNext(parser); reader.parseSkipOne(); } else { if (id==2) { long value = reader.capturedLongField(reader, 1); StringBuilder builder = new StringBuilder(); reader.capturedFieldBytesAsUTF8(reader, 2, builder); if (id!=1) { long value = reader.capturedLongField(reader, 1);
void convertMQTTTopicsToLocal(final byte[] backing1, final int pos1, final int len1, final int mask1) { int size = Pipe.addMsgIdx(tempSubject, RawDataSchema.MSG_CHUNKEDSTREAM_1); DataOutputBlobWriter<RawDataSchema> stream = Pipe.outputStream(tempSubject); DataOutputBlobWriter.openField(stream); TrieParserReader.parseSetup(TrieParserReaderLocal.get(), backing1, pos1, len1, mask1); boolean foundEnd = false; while (TrieParserReader.parseHasContent(TrieParserReaderLocal.get())) { if (foundEnd) { throw new UnsupportedOperationException("Invalid topic if /# is used it must only be at the end."); } int token = (int)TrieParserReader.parseNext(TrieParserReaderLocal.get(), topicConversionTrie); if (-1 == token) { stream.write(TrieParserReader.parseSkipOne(TrieParserReaderLocal.get())); } if (1 == token) { stream.write(MessagePubSubImpl.WILD_POUND_THE_END); // /# foundEnd = true; } if (2 == token) { stream.write(MessagePubSubImpl.WILD_PLUS_THE_SEGMENT); // +/ } } DataOutputBlobWriter.closeLowLevelField(stream); Pipe.confirmLowLevelWrite(tempSubject, size); Pipe.publishWrites(tempSubject); }
private void parseExtractionPath(String extractionPath) { JSONExtractorActive running = this; TrieParserReader trieParserReader = TrieParserReaderLocal.get(); CharSequenceToUTF8Local.get() .convert(extractionPath) .append(".") .parseSetup(trieParserReader); int token = 0; while ((token = (int)trieParserReader.parseNext(parser)) != -1) { switch(token) { case keyToken: path.add(TrieParserReader.capturedFieldBytesAsUTF8(trieParserReader, 0, new StringBuilder())); break; case dotToken: break; case arrayToken: path.add("[]"); break; } } }
TrieParserReader.parseSetup(reader, logFileInput); files = new ArrayList<byte[]>(); while ( (id=reader.parseNext(lineParser))>=0 ) { TrieParserReader.capturedFieldBytesAsUTF8(reader, 0, accum); accum.append('\n');
if (!TrieParserReader.parseHasContent(templateParserReader)) { logger.info("the leading / was added on URL since route did not define it"); outputStream.writeByte('/'); while(TrieParserReader.parseHasContent(templateParserReader)) { long token = TrieParserReader.parseNext(templateParserReader, templateParser); assert(token<=255) : "type must fit into 8 bits"; assert(token>=-1); TrieParserReader.capturedFieldSetValue(templateParserReader, 0, runtimeParser, (token<<16) | (fieldIndex++));//type high 16, and position in low 16 TrieParserReader.capturedFieldSetValue(templateParserReader, 0, runtimeParser, (token<<16) | (fieldIndex++));//type high 16, and position in low 16 TrieParserReader.capturedFieldSetValue(templateParserReader, 0, runtimeParser, (token<<16) | (fieldIndex++));//type high 16, and position in low 16 TrieParserReader.capturedFieldSetValue(templateParserReader, 0, runtimeParser, (token<<16) | (fieldIndex++));//type high 16, and position in low 16 case -1: int value = TrieParserReader.parseSkipOne(templateParserReader); if (value>=0) { if (('/'!=(char)value) && (0 == outputStream.position())) {
int startingLength1 = TrieParserReader.savePositionMemo(trieReader, positionMemoData, memoIdx); if (startingLength1 >= (revisionMap.shortestKnown()+1)) { TrieParserReader.setCapturedShort(trieReader, 0, 200); final int revisionId = (int)TrieParserReader.parseNext(trieReader, revisionMap, -1, -2); if (revisionId>=0) { processFoundRevision(i, stateIdx, targetPipe, ccId, cc, startingLength1); } else { assert(trieReader.sourceLen <= trieReader.sourceMask) : "ERROR the source length is larger than the backing array"; TrieParserReader.loadPositionMemo(trieReader, positionMemoData, memoIdx); TrieParserReader.loadPositionMemo(trieReader, positionMemoData, memoIdx); final int consumed = TrieParserReader.parseCopy(trieReader, Math.min(lengthRemaining, DataOutputBlobWriter.lastBackPositionOfIndex(writer2)), TrieParserReader.savePositionMemo(trieReader, positionMemoData, memoIdx); TrieParserReader.savePositionMemo(trieReader, positionMemoData, memoIdx); } else { foundWork = 0;//we must exit to give the other stages a chance to fix this issue int startingLength3 = TrieParserReader.savePositionMemo(trieReader, positionMemoData, memoIdx); if ((int)TrieParserReader.parseNext(trieReader, HTTPUtil.chunkMap) < 0) {
assert(Arrays.equals(HTTPHeaderDefaults.CONTENT_LENGTH.rootBytes(),header.rootBytes())) : "Custom enums must share same ordinal positions, CONTENT_LENGTH does not match"; long length = TrieParserReader.capturedLongField(trieReader, 0); assert(Arrays.equals(HTTPHeaderDefaults.CONTENT_TYPE.rootBytes(),header.rootBytes())) : "Custom enums must share same ordinal positions, CONTENT_TYPE does not match"; int type = (int)TrieParserReader.capturedFieldQuery(trieReader, 0, trieReaderCharType, httpSpec.contentTypeTrieBuilder()); if (TrieParserReader.hasCapturedBytes(trieReaderCharType,0)) { String encoding = trieReader.capturedFieldBytesAsUTF8(trieReaderCharType, 0, new StringBuilder()).toString(); logger.info("encoding detected of {}",encoding); closeRequested[i] = TrieParserReader.capturedFieldBytesEquals(trieReader, 0, CLOSE, 0, Integer.MAX_VALUE); TrieParserReader.writeCapturedValuesToDataOutput(trieReader, writer); } else { TrieParserReader.writeCapturedValuesToDataOutput(trieReader, writer);
private TrieParserReader parserReader() { if (null == reader) { reader = new TrieParserReader(true); } return reader; }
while ((remainingLen=TrieParserReader.parseHasContentLength(trieReader))>0){ long headerToken = TrieParserReader.parseNext(trieReader, headerMap); assert(Arrays.equals(HTTPHeaderDefaults.CONTENT_LENGTH.rootBytes(),header.rootBytes())) : "Custom enums must share same ordinal positions, CONTENT_LENGTH does not match"; postLength = TrieParserReader.capturedLongField(trieReader, 0); } else if (HTTPHeaderDefaults.TRANSFER_ENCODING.ordinal() == header.ordinal()) { assert(Arrays.equals(HTTPHeaderDefaults.TRANSFER_ENCODING.rootBytes(),header.rootBytes())) : "Custom enums must share same ordinal positions, TRANSFER_ENCODING does not match"; TrieParserReader.writeCapturedValuesToDataOutput(trieReader, writer); DataOutputBlobWriter.setIntBackData(writer, writePosition, StructRegistry.FIELD_MASK & (int)headerToken);
private void parseErrorWhileChunking(final int memoIdx, Pipe<NetPayloadSchema> pipe, int readingPos) { System.err.println("SHUTING DOWN NOW: chunk ID is TOO long starting at "+readingPos+" data remaining "+trieReader.sourceLen); ByteArrayOutputStream ist = new ByteArrayOutputStream(); trieReader.debugAsUTF8(trieReader, new PrintStream(ist), 100,false); byte[] data = ist.toByteArray(); System.err.println(new String(data)); System.err.println("bad data pipe is:"+pipe); trieReader.debug(); //failure position is AT the mask?? TrieParserReader.loadPositionMemo(trieReader, positionMemoData, memoIdx); int chunkId3 = (int)TrieParserReader.parseNext(trieReader, HTTPUtil.chunkMap); System.err.println("parsed value was "+chunkId3); requestShutdown(); }
public void parseSetup(TrieParserReader trieParserReader) { TrieParserReader.parseSetup(trieParserReader, backing, 0, length, Integer.MAX_VALUE); }
public static int debugAsUTF8(TrieParserReader that, Appendable target) { return debugAsUTF8(that,target, Integer.MAX_VALUE); } public static int debugAsUTF8(TrieParserReader that, Appendable target, int maxLen) {