/** Fills virtual 'start' arc, ie, an empty incoming arc to * the FST's start node */ public Arc<T> getFirstArc(Arc<T> arc) { T NO_OUTPUT = outputs.getNoOutput(); if (emptyOutput != null) { arc.flags = BIT_FINAL_ARC | BIT_LAST_ARC; arc.nextFinalOutput = emptyOutput; if (emptyOutput != NO_OUTPUT) { arc.flags |= BIT_ARC_HAS_FINAL_OUTPUT; } } else { arc.flags = BIT_LAST_ARC; arc.nextFinalOutput = NO_OUTPUT; } arc.output = NO_OUTPUT; // If there are no nodes, ie, the FST only accepts the // empty string, then startNode is 0 arc.target = startNode; return arc; }
private boolean valid(Pair<A,B> pair) { final boolean noOutput1 = pair.output1.equals(outputs1.getNoOutput()); final boolean noOutput2 = pair.output2.equals(outputs2.getNoOutput()); if (noOutput1 && pair.output1 != outputs1.getNoOutput()) { return false; } if (noOutput2 && pair.output2 != outputs2.getNoOutput()) { return false; } if (noOutput1 && noOutput2) { if (pair != NO_OUTPUT) { return false; } else { return true; } } else { return true; } }
public PairOutputs(Outputs<A> outputs1, Outputs<B> outputs2) { this.outputs1 = outputs1; this.outputs2 = outputs2; NO_OUTPUT = new Pair<>(outputs1.getNoOutput(), outputs2.getNoOutput()); }
/** Create a new Pair */ public Pair<A,B> newPair(A a, B b) { if (a.equals(outputs1.getNoOutput())) { a = outputs1.getNoOutput(); } if (b.equals(outputs2.getNoOutput())) { b = outputs2.getNoOutput(); } if (a == outputs1.getNoOutput() && b == outputs2.getNoOutput()) { return NO_OUTPUT; } else { final Pair<A,B> p = new Pair<>(a, b); assert valid(p); return p; } }
private long ramBytesUsed(Arc<T>[] arcs) { long size = 0; if (arcs != null) { size += RamUsageEstimator.shallowSizeOf(arcs); for (Arc<T> arc : arcs) { if (arc != null) { size += ARC_SHALLOW_RAM_BYTES_USED; if (arc.output != null && arc.output != outputs.getNoOutput()) { size += outputs.ramBytesUsed(arc.output); } if (arc.nextFinalOutput != null && arc.nextFinalOutput != outputs.getNoOutput()) { size += outputs.ramBytesUsed(arc.nextFinalOutput); } } } } return size; }
/** Adds all leaving arcs, including 'finished' arc, if * the node is final, from this node into the queue. */ public void addStartPaths(FST.Arc<T> node, T startOutput, boolean allowEmptyString, IntsRefBuilder input, float boost, CharSequence context, int payload) throws IOException { // De-dup NO_OUTPUT since it must be a singleton: if (startOutput.equals(fst.outputs.getNoOutput())) { startOutput = fst.outputs.getNoOutput(); } FSTPath<T> path = new FSTPath<>(startOutput, node, input, boost, context, payload); fst.readFirstTargetArc(node, path.arc, bytesReader); // Bootstrap: find the min starting arc while (true) { if (allowEmptyString || path.arc.label != FST.END_LABEL) { addIfCompetitive(path); } if (path.arc.isLast()) { break; } fst.readNextArc(path.arc, bytesReader); } }
/** doFloor controls the behavior of advance: if it's true * doFloor is true, advance positions to the biggest * term before target. */ protected FSTEnum(FST<T> fst) { this.fst = fst; fstReader = fst.getBytesReader(); NO_OUTPUT = fst.outputs.getNoOutput(); fst.getFirstArc(getArc(0)); output[0] = NO_OUTPUT; }
final T NO_OUTPUT = fst.outputs.getNoOutput(); final BytesReader r = fst.getBytesReader();
dedupHash = null; NO_OUTPUT = outputs.getNoOutput();
/** Looks up the output for this input, or null if the * input is not accepted */ public static<T> T get(FST<T> fst, BytesRef input) throws IOException { assert fst.inputType == FST.INPUT_TYPE.BYTE1; final BytesReader fstReader = fst.getBytesReader(); // TODO: would be nice not to alloc this on every lookup final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>()); // Accumulate output as we go T output = fst.outputs.getNoOutput(); for(int i=0;i<input.length;i++) { if (fst.findTargetArc(input.bytes[i+input.offset] & 0xFF, arc, arc, fstReader) == null) { return null; } output = fst.outputs.add(output, arc.output); } if (arc.isFinal()) { return fst.outputs.add(output, arc.nextFinalOutput); } else { return null; } }
/** Looks up the output for this input, or null if the * input is not accepted. */ public static<T> T get(FST<T> fst, IntsRef input) throws IOException { // TODO: would be nice not to alloc this on every lookup final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>()); final BytesReader fstReader = fst.getBytesReader(); // Accumulate output as we go T output = fst.outputs.getNoOutput(); for(int i=0;i<input.length;i++) { if (fst.findTargetArc(input.ints[input.offset + i], arc, arc, fstReader) == null) { return null; } output = fst.outputs.add(output, arc.output); } if (arc.isFinal()) { return fst.outputs.add(output, arc.nextFinalOutput); } else { return null; } }
final T NO_OUTPUT = fst.outputs.getNoOutput();
arc.output = outputs.read(in); } else { arc.output = outputs.getNoOutput(); arc.nextFinalOutput = outputs.readFinalOutput(in); } else { arc.nextFinalOutput = outputs.getNoOutput();
long addNode(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws IOException { T NO_OUTPUT = outputs.getNoOutput();
/** * Returns the value mapped to the given key or <code>null</code> if the key is not in the FST dictionary. */ public BytesRef get(char[] buffer, int bufferLen, Arc<BytesRef> scratchArc, BytesReader fstReader) throws IOException { BytesRef pendingOutput = fst.outputs.getNoOutput(); BytesRef matchOutput = null; int bufUpto = 0; fst.getFirstArc(scratchArc); while (bufUpto < bufferLen) { final int codePoint = Character.codePointAt(buffer, bufUpto, bufferLen); if (fst.findTargetArc(ignoreCase ? Character.toLowerCase(codePoint) : codePoint, scratchArc, scratchArc, fstReader) == null) { return null; } pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output); bufUpto += Character.charCount(codePoint); } if (scratchArc.isFinal()) { matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput); } return matchOutput; }
final FST.Arc<IntsRef> arc = fst.getFirstArc(new FST.Arc<IntsRef>()); final IntsRef NO_OUTPUT = fst.outputs.getNoOutput(); IntsRef output = NO_OUTPUT;
static void applyMappings(FST<CharsRef> fst, StringBuilder sb) throws IOException { final FST.BytesReader bytesReader = fst.getBytesReader(); final FST.Arc<CharsRef> firstArc = fst.getFirstArc(new FST.Arc<CharsRef>()); final CharsRef NO_OUTPUT = fst.outputs.getNoOutput();
int matchEndOffset = -1; BytesRef pendingOutput = fst.outputs.getNoOutput(); fst.getFirstArc(scratchArc); assert scratchArc.output == fst.outputs.getNoOutput();
int matchInputLength = 0; BytesRef pendingOutput = fst.outputs.getNoOutput(); fst.getFirstArc(scratchArc); assert scratchArc.output == fst.outputs.getNoOutput();
FST.Arc<IntsRef> arc = prefixArcs[recursionDepth]; fst.getFirstArc(arc); IntsRef NO_OUTPUT = outputs.getNoOutput(); IntsRef output = NO_OUTPUT; int limit = dictionary.fullStrip ? length : length-1; FST.Arc<IntsRef> arc = suffixArcs[recursionDepth]; fst.getFirstArc(arc); IntsRef NO_OUTPUT = outputs.getNoOutput(); IntsRef output = NO_OUTPUT; int limit = dictionary.fullStrip ? 0 : 1;