public StepSeekingIterator(SeekableLineReader slr, long startOffset, long endOffset, int numSplits) throws IOException { this.slr = slr; this.currSplit = 0; this.startOffset = startOffset; this.numSplits = numSplits; this.endOffset = endOffset; slr.seek(startOffset); }
public StepSeekingIterator(SeekableLineReader slr, long startOffset, long endOffset, int numSplits) throws IOException { this.slr = slr; this.currSplit = 0; this.startOffset = startOffset; this.numSplits = numSplits; this.endOffset = endOffset; slr.seek(startOffset); }
public StepSeekingIterator(SeekableLineReader slr, long startOffset, long endOffset, int numSplits) throws IOException { this.slr = slr; this.currSplit = 0; this.startOffset = startOffset; this.numSplits = numSplits; this.endOffset = endOffset; slr.seek(startOffset); }
@Override public void seek(long offset) throws IOException { this.slr.seek(offset); }
@Override public void seek(long offset) throws IOException { this.slr.seek(offset); }
@Override public void seek(long offset) throws IOException { this.slr.seek(offset); }
public String next() { String line = null; try { if (startOffset + currSplit != 0) { slr.skipLine(); } line = slr.readLine(); currSplit++; long seekDiff = ((endOffset - startOffset) * currSplit) / numSplits; slr.seek(startOffset + seekDiff); } catch (IOException io) { io.printStackTrace(); } return line; }
public CloseableIterator<String> getRecordIterator(final long offset) throws IOException { SeekableLineReader slr = factory.get(); slr.seek(offset); return new SeekableLineReaderIterator(slr); }
public String next() { String line = null; try { if (startOffset + currSplit != 0) { slr.skipLine(); } line = slr.readLine(); currSplit++; long seekDiff = ((endOffset - startOffset) * currSplit) / numSplits; slr.seek(startOffset + seekDiff); } catch (IOException io) { io.printStackTrace(); } return line; }
public CloseableIterator<String> getRecordIterator(final long offset) throws IOException { SeekableLineReader slr = factory.get(); slr.seek(offset); return new SeekableLineReaderIterator(slr); }
public CloseableIterator<String> getRecordIterator(final long offset) throws IOException { SeekableLineReader slr = factory.get(); slr.seek(offset); return new SeekableLineReaderIterator(slr); }
public String next() { String line = null; try { if (startOffset + currSplit != 0) { slr.skipLine(); } line = slr.readLine(); currSplit++; long seekDiff = ((endOffset - startOffset) * currSplit) / numSplits; slr.seek(startOffset + seekDiff); } catch (IOException io) { io.printStackTrace(); } return line; }
private long searchOffset(SeekableLineReader slr, final String key, boolean lessThan, Comparator<String> comparator) throws IOException { long offset = binaryFindOffset(slr, key, comparator); slr.seek(offset); String line = null; if (offset > 0) { slr.skipLine(); } String prev = null; while(true) { if (line != null) { offset += line.getBytes().length + 1; } line = slr.readLine(); if(line == null) break; if(comparator.compare(line, key) >= 0) break; prev = line; } if (lessThan && prev != null) { offset -= prev.getBytes().length + 1; } return offset; }
private long searchOffset(SeekableLineReader slr, final String key, boolean lessThan, Comparator<String> comparator) throws IOException { long offset = binaryFindOffset(slr, key, comparator); slr.seek(offset); String line = null; if (offset > 0) { slr.skipLine(); } String prev = null; while(true) { if (line != null) { offset += line.getBytes().length + 1; } line = slr.readLine(); if(line == null) break; if(comparator.compare(line, key) >= 0) break; prev = line; } if (lessThan && prev != null) { offset -= prev.getBytes().length + 1; } return offset; }
private CloseableIterator<String> search(SeekableLineReader slr, final String key, boolean lessThan, Comparator<String> comparator) throws IOException { long min = binaryFindOffset(slr, key, comparator); if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine(String.format("Aligning(%d)",min)); } slr.seek(min); String line; if (min > 0) { slr.skipLine(); } String prev = null; while(true) { line = slr.readLine(); if (line == null) break; if (comparator.compare(line, key) >= 0) break; prev = line; } if (!lessThan) { prev = null; } return new CachedStringIterator(slr, prev, line); }
private CloseableIterator<String> search(SeekableLineReader slr, final String key, boolean lessThan, Comparator<String> comparator) throws IOException { long min = binaryFindOffset(slr, key, comparator); if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine(String.format("Aligning(%d)",min)); } slr.seek(min); String line; if (min > 0) { slr.skipLine(); } String prev = null; while(true) { line = slr.readLine(); if (line == null) break; if (comparator.compare(line, key) >= 0) break; prev = line; } if (!lessThan) { prev = null; } return new CachedStringIterator(slr, prev, line); }
public String getLastLine(SeekableLineReader slr) throws IOException { int lastLineLenTest = 0; int lastLineLenInc = 400; String endLine = null; do { lastLineLenTest += lastLineLenInc; slr.seek(slr.getSize() - lastLineLenTest); // TODO: assume larger buffer slr.readLine(); // skip partial line String nextLine = null; endLine = null; while ((nextLine = slr.readLine()) != null) { endLine = nextLine; } } while (endLine == null); return endLine; }
public String getLastLine(SeekableLineReader slr) throws IOException { int lastLineLenTest = 0; int lastLineLenInc = 400; String endLine = null; do { lastLineLenTest += lastLineLenInc; slr.seek(slr.getSize() - lastLineLenTest); // TODO: assume larger buffer slr.readLine(); // skip partial line String nextLine = null; endLine = null; while ((nextLine = slr.readLine()) != null) { endLine = nextLine; } } while (endLine == null); return endLine; }
public String getLastLine(SeekableLineReader slr) throws IOException { int lastLineLenTest = 0; int lastLineLenInc = 400; String endLine = null; do { lastLineLenTest += lastLineLenInc; slr.seek(slr.getSize() - lastLineLenTest); // TODO: assume larger buffer slr.readLine(); // skip partial line String nextLine = null; endLine = null; while ((nextLine = slr.readLine()) != null) { endLine = nextLine; } } while (endLine == null); return endLine; }
slr.seek(0); startLine = slr.readLine(); } else {