public void init() throws IOException { if (summaryFile != null) { this.summary = new SortedTextFile(GeneralURIStreamFactory.createSeekableStreamFactory(summaryFile, readaheadSize, useNio)); this.summary.setBinsearchBlockSize(binsearchBlockSize); } if (blockLoader == null) { this.blockLoader = new ZipNumBlockLoader(); } }
public CloseableIterator<String> getRecordIteratorLT(final String prefix) throws IOException { return getRecordIterator(prefix, true); }
public CloseableIterator<String> getSplitIterator(String start, String end, int numSplits) throws IOException { SeekableLineReader slr = factory.get(); long[] offsets = getStartEndOffsets(slr, start, end); return new StepSeekingIterator(slr, offsets[0], offsets[1], numSplits); }
SortedTextFile sorted = new SortedTextFile(factory); sorted.setBinsearchBlockSize(blocksize); CloseableIterator<String> iter = sorted.getRecordIterator(next); if (iter.hasNext()) { System.out.println(iter.next());
slr = summary.getSLR(); long[] offsets = summary.getStartEndOffsets(slr, start, end); endLine = summary.getLastLine(slr);
LOGGER.warning("Added file: " + spec); SortedTextFile stf = new SortedTextFile(factory); mergeItr.addIterator(stf.getRecordIterator(key, lessThan));
public long computeTotalLines() { long numLines = 0; try { numLines = this.getNumLines(summary.getRange("", "")); } catch (IOException e) { LOGGER.warning(e.toString()); return 0; } long adjustment = getTotalAdjustment(); numLines -= (getNumBlocks() - 1); numLines *= this.getCdxLinesPerBlock(); numLines += adjustment; return numLines; }
public CloseableIterator<String> getCDXIterator(String key, ZipNumParams params) throws IOException { CloseableIterator<String> summaryIter = summary.getRecordIteratorLT(key); return wrapStartIterator(getCDXIterator(summaryIter, params), key); }
startLine = slr.readLine(); } else { startLine = search(slr, start, true, defaultComparator).next(); endLine = getLastLine(slr); } else { endLine = search(slr, end, true, defaultComparator).next();
slr = summary.getSLR(); FieldExtractingSLR lineCountReader = new FieldExtractingSLR(slr, lineField, "\t"); long offset = summary.binaryFindOffset(lineCountReader, lineNumber, SortedTextFile.numericComparator); slr.seek(offset);
slr = factory.get(); long[] offsets = getStartEndOffsets(slr, start, end); long startOffset = offsets[0]; long diff = offsets[1] - offsets[0]; endLine = getLastLine(slr);
private long searchOffset(SeekableLineReader slr, final String key, boolean lessThan, Comparator<String> comparator) throws IOException { long offset = binaryFindOffset(slr, key, comparator); slr.seek(offset); String line = null; if (offset > 0) { slr.skipLine(); } String prev = null; while(true) { if (line != null) { offset += line.getBytes().length + 1; } line = slr.readLine(); if(line == null) break; if(comparator.compare(line, key) >= 0) break; prev = line; } if (lessThan && prev != null) { offset -= prev.getBytes().length + 1; } return offset; }
public void setPathIndex(String path) throws IOException { this.path = path; this.pathIndex = new SortedTextFile(path, false); }
public CloseableIterator<String> getRecordIterator(final String prefix, boolean lessThan) throws IOException { SeekableLineReader slr = factory.get(); try { return search(slr, prefix, lessThan, defaultComparator); } catch (IOException io) { if (slr != null) { slr.close(); } throw io; } }
@Override public void run() { try { while (true) { long currModTime = locReaderFactory.getModTime(); if (currModTime != lastModTime) { syncLoad(currModTime); Thread.sleep(checkInterval); if (summary != null) { summary.reloadFactory(); } } Thread.sleep(checkInterval); } } catch (InterruptedException ie) { } } }
SortedTextFile sorted = new SortedTextFile(factory); sorted.setBinsearchBlockSize(blocksize); CloseableIterator<String> iter = sorted.getRecordIterator(next); if (iter.hasNext()) { System.out.println(iter.next());
slr = summary.getSLR(); long[] offsets = summary.getStartEndOffsets(slr, start, end); endLine = summary.getLastLine(slr);
LOGGER.warning("Added file: " + spec); SortedTextFile stf = new SortedTextFile(factory); mergeItr.addIterator(stf.getRecordIterator(key, lessThan));
public long computeTotalLines() { long numLines = 0; try { numLines = this.getNumLines(summary.getRange("", "")); } catch (IOException e) { LOGGER.warning(e.toString()); return 0; } long adjustment = getTotalAdjustment(); numLines -= (getNumBlocks() - 1); numLines *= this.getCdxLinesPerBlock(); numLines += adjustment; return numLines; }
public CloseableIterator<String> getCDXIterator(String key, ZipNumParams params) throws IOException { CloseableIterator<String> summaryIter = summary.getRecordIteratorLT(key); return wrapStartIterator(getCDXIterator(summaryIter, params), key); }