Refine search
private boolean assertRootCachedArc(int label, Arc<T> cachedArc) throws IOException { Arc<T> arc = new Arc<>(); getFirstArc(arc); BytesReader in = getBytesReader(); Arc<T> result = findTargetArc(label, arc, arc, in, false); if (result == null) { assert cachedArc == null; } else { assert cachedArc != null; assert cachedArc.arcIdx == result.arcIdx; assert cachedArc.bytesPerArc == result.bytesPerArc; assert cachedArc.flags == result.flags; assert cachedArc.label == result.label; assert cachedArc.nextArc == result.nextArc; assert cachedArc.nextFinalOutput.equals(result.nextFinalOutput); assert cachedArc.numArcs == result.numArcs; assert cachedArc.output.equals(result.output); assert cachedArc.posArcsStart == result.posArcsStart; assert cachedArc.target == result.target; } return true; }
/** * Reads an automaton from a file. */ public static <T> FST<T> read(Path path, Outputs<T> outputs) throws IOException { try (InputStream is = Files.newInputStream(path)) { return new FST<>(new InputStreamDataInput(new BufferedInputStream(is)), outputs); } }
final PendingBlock root = (PendingBlock) pending.get(0); assert root.prefix.length == 0; assert root.index.getEmptyOutput() != null; root.index.save(indexOut); ((PendingBlock) pending.get(0)).index.getEmptyOutput(), numTerms, indexStartFP,
/** In-place read; returns the arc. */ public Arc<T> readNextArc(Arc<T> arc, BytesReader in) throws IOException { if (arc.label == END_LABEL) { // This was a fake inserted "final" arc if (arc.nextArc <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } return readFirstRealTargetArc(arc.nextArc, arc, in); } else { return readNextRealArc(arc, in); } }
/** doFloor controls the behavior of advance: if it's true * doFloor is true, advance positions to the biggest * term before target. */ protected FSTEnum(FST<T> fst) { this.fst = fst; fstReader = fst.getBytesReader(); NO_OUTPUT = fst.outputs.getNoOutput(); fst.getFirstArc(getArc(0)); output[0] = NO_OUTPUT; }
final Arc<T> arc = new Arc<>(); getFirstArc(arc); if (targetHasArcs(arc)) { final BytesReader in = getBytesReader(); Arc<T>[] arcs = (Arc<T>[]) new Arc[0x80]; readFirstRealTargetArc(arc.target, arc, in); int count = 0; while(true) { assert arc.label != END_LABEL; if (arc.label < arcs.length) { arcs[arc.label] = new Arc<T>().copyFrom(arc); } else { break; break; readNextRealArc(arc, in); count++; int cacheRAM = (int) ramBytesUsed(arcs); if (count >= FIXED_ARRAY_NUM_ARCS_SHALLOW && cacheRAM < ramBytesUsed()/5) { cachedRootArcs = arcs; cachedArcsBytesUsed = cacheRAM;
assert arc.isFinal(); output = arc.output; targetUpto = 0; output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output); if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); assert arc.output != null; final FST.Arc<BytesRef> nextArc = fr.index.findTargetArc(targetLabel, arc, getArc(1+targetUpto), fstReader);
final FST.BytesReader in = fst.getBytesReader(); int low = arc.arcIdx; int high = arc.numArcs-1; in.setPosition(arc.posArcsStart); in.skipBytes(arc.bytesPerArc*mid+1); final int midLabel = fst.readLabel(in); final int cmp = midLabel - targetLabel; fst.readNextRealArc(arc, in); assert arc.arcIdx == mid; assert arc.label == targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel + " mid=" + mid; arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); continue; fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); if (arc.label < targetLabel) { while(!arc.isLast() && fst.readNextArcLabel(arc, in) < targetLabel) { fst.readNextArc(arc, fstReader); fst.readNextRealArc(arc, in); assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel; pushLast(); arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader);
int targetLabel = getTargetLabel(); final FST.BytesReader fstReader = fst.getBytesReader(); final FST.Arc<T> nextArc = fst.findTargetArc(targetLabel, arc, getArc(upto), fstReader); if (nextArc == null) { fst.readFirstTargetArc(arc, getArc(upto), fstReader);
if (follow.isFinal()) { if (follow.target <= 0) { arc.flags = FST.BIT_LAST_ARC; if (!FST.targetHasArcs(follow)) { return null; fst.readFirstTargetArc(follow, arc, in); if (arc.bytesPerArc != 0 && arc.label != FST.END_LABEL) { in.setPosition(arc.posArcsStart); in.skipBytes(arc.bytesPerArc * mid + 1); final int midLabel = fst.readLabel(in); final int cmp = midLabel - label; } else { arc.arcIdx = mid-1; return fst.readNextRealArc(arc, in); return fst.readNextRealArc(arc, in); fst.readFirstRealTargetArc(follow.target, arc, in); } else if (arc.isLast()) { return null; } else { fst.readNextRealArc(arc, in);
/** * Writes an automaton to a file. */ public void save(final Path path) throws IOException { try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) { save(new OutputStreamDataOutput(os)); } }
final BytesReader fstReader = fst.getBytesReader(); final T NO_OUTPUT = fst.outputs.getNoOutput(); fst.readFirstTargetArc(path.arc, path.arc, fstReader); break; } else if (!foundZero) { scratchArc.copyFrom(path.arc); foundZero = true; } else { addIfCompetitive(path); if (path.arc.isLast()) { break; fst.readNextArc(path.arc, fstReader); path.arc.copyFrom(scratchArc);
BytesReader bytesReader = fst.getBytesReader(); FST.Arc<Pair<Long,BytesRef>> scratchArc = new FST.Arc<>(); if (fst.findTargetArc(endByte, path.fstNode, scratchArc, bytesReader) != null) { if (fst.findTargetArc(endByte, path.fstNode, scratchArc, bytesReader) != null) {
@Override public long ramBytesUsed() { return BASE_RAM_BYTES_USED + ((index!=null)? index.ramBytesUsed() : 0); }
stats.indexNumBytes = fr.index.ramBytesUsed(); FST.Arc<BytesRef> arc; if (fr.index != null) { arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); } else { arc = null; arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); } else { arc = null;
/** * Cache the root node's output arcs starting with completions with the highest weights. */ @SuppressWarnings("unchecked") private void cacheRootArcs() throws IOException { if (automaton != null) { List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>(); Arc<Object> arc = automaton.getFirstArc(new Arc<Object>()); automaton.readFirstTargetArc(arc, arc); while (true) { rootArcs.add(new Arc<Object>().copyFrom(arc)); if (arc.isLast()) break; automaton.readNextArc(arc); } Collections.reverse(rootArcs); // we want highest weights first. this.rootArcs = rootArcs.toArray(new Arc[rootArcs.size()]); } }
public TopNSearcher(FST<T> fst, int topN, int maxQueueDepth, Comparator<T> comparator, Comparator<FSTPath<T>> pathComparator) { this.fst = fst; this.bytesReader = fst.getBytesReader(); this.topN = topN; this.maxQueueDepth = maxQueueDepth; this.comparator = comparator; this.pathComparator = pathComparator; queue = new TreeSet<>(pathComparator); }
/** Finds an arc leaving the incoming arc, replacing the arc in place. * This returns null if the arc was not found, else the incoming arc. */ public Arc<T> findTargetArc(int labelToMatch, Arc<T> follow, Arc<T> arc, BytesReader in) throws IOException { return findTargetArc(labelToMatch, follow, arc, in, true); }
FST.Arc<CharsRef> arc = cachedRootArcs.get(Character.valueOf((char) firstCH)); if (arc != null) { if (!FST.targetHasArcs(arc)) { assert arc.isFinal(); lastMatchLen = 1; lastMatch = arc.output; lookahead++; if (arc.isFinal()) { if (!FST.targetHasArcs(arc)) { break; break; if ((arc = map.findTargetArc(ch, arc, scratchArc, fstReader)) == null) {
protected void doNext() throws IOException { //System.out.println("FE: next upto=" + upto); if (upto == 0) { //System.out.println(" init"); upto = 1; fst.readFirstTargetArc(getArc(0), getArc(1), fstReader); } else { // pop //System.out.println(" check pop curArc target=" + arcs[upto].target + " label=" + arcs[upto].label + " isLast?=" + arcs[upto].isLast()); while (arcs[upto].isLast()) { upto--; if (upto == 0) { //System.out.println(" eof"); return; } } fst.readNextArc(arcs[upto], fstReader); } pushFirst(); }