mDataFileReader.seek(indexEntry.getValue());
@Override public void reopen(FileInputSplit split, Tuple2<Long, Long> state) throws IOException { Preconditions.checkNotNull(split, "reopen() cannot be called on a null split."); Preconditions.checkNotNull(state, "reopen() cannot be called with a null initial state."); try { this.open(split); } finally { if (state.f0 != -1) { lastSync = state.f0; recordsReadSinceLastSync = state.f1; } } if (lastSync != -1) { // open and read until the record we were before // the checkpoint and discard the values dataFileReader.seek(lastSync); for (int i = 0; i < recordsReadSinceLastSync; i++) { dataFileReader.next(null); } } } }
/** * Construct a reader for a file at the current position of the input, * without reading the header. * @param sync True to read forward to the next sync point after opening, * false to assume that the input is already at a valid sync * point. */ public static <D> DataFileReader<D> openReader(SeekableInput in, DatumReader<D> reader, Header header, boolean sync) throws IOException { DataFileReader<D> dreader = new DataFileReader<D>(in, reader, header); // seek/sync to an (assumed) valid position if (sync) dreader.sync(in.tell()); else dreader.seek(in.tell()); return dreader; }
/** * Construct a reader for a file at the current position of the input, * without reading the header. * @param sync True to read forward to the next sync point after opening, * false to assume that the input is already at a valid sync * point. */ public static <D> DataFileReader<D> openReader(SeekableInput in, DatumReader<D> reader, Header header, boolean sync) throws IOException { DataFileReader<D> dreader = new DataFileReader<>(in, reader, header); // seek/sync to an (assumed) valid position if (sync) dreader.sync(in.tell()); else dreader.seek(in.tell()); return dreader; }
seek(position);
seek(position);
public void testSyncDiscovery() throws IOException { File file = makeFile(); DataFileReader<Object> reader = new DataFileReader<>(file, new GenericDatumReader<>()); try { // discover the sync points ArrayList<Long> syncs = new ArrayList<>(); long previousSync = -1; while (reader.hasNext()) { if (reader.previousSync() != previousSync) { previousSync = reader.previousSync(); syncs.add(previousSync); } reader.next(); } // confirm that the first point is the one reached by sync(0) reader.sync(0); assertEquals((long)reader.previousSync(), (long)syncs.get(0)); // and confirm that all points are reachable for (Long sync : syncs) { reader.seek(sync); assertNotNull(reader.next()); } } finally { reader.close(); } }
new DataFileReader<>(file, new GenericDatumReader<>()); try { reader.seek(start); Object datum = null; if (VALIDATE) {
dataFileReader.seek(indexRecords.get(0).getValue()); assertTrue(dataFileReader.hasNext()); AvroKeyValue<CharSequence, CharSequence> appleRecord assertEquals("Apple", appleRecord.getValue().toString()); dataFileReader.seek(indexRecords.get(1).getValue()); assertTrue(dataFileReader.hasNext()); AvroKeyValue<CharSequence, CharSequence> carrotRecord
seek(position); try { int i=0, b;
seek(position); try { int i=0, b;
/** * Construct a reader for a file at the current position of the input, * without reading the header. * @param sync True to read forward to the next sync point after opening, * false to assume that the input is already at a valid sync * point. */ public static <D> DataFileReader<D> openReader(SeekableInput in, DatumReader<D> reader, Header header, boolean sync) throws IOException { DataFileReader<D> dreader = new DataFileReader<D>(in, reader, header); // seek/sync to an (assumed) valid position if (sync) dreader.sync(in.tell()); else dreader.seek(in.tell()); return dreader; }
@Override public void reopen(FileInputSplit split, Tuple2<Long, Long> state) throws IOException { Preconditions.checkNotNull(split, "reopen() cannot be called on a null split."); Preconditions.checkNotNull(state, "reopen() cannot be called with a null initial state."); try { this.open(split); } finally { if (state.f0 != -1) { lastSync = state.f0; recordsReadSinceLastSync = state.f1; } } if (lastSync != -1) { // open and read until the record we were before // the checkpoint and discard the values dataFileReader.seek(lastSync); for (int i = 0; i < recordsReadSinceLastSync; i++) { dataFileReader.next(null); } } } }
/** * Construct a reader for a file at the current position of the input, * without reading the header. * @param sync True to read forward to the next sync point after opening, * false to assume that the input is already at a valid sync * point. */ public static <D> DataFileReader<D> openReader(SeekableInput in, DatumReader<D> reader, Header header, boolean sync) throws IOException { DataFileReader<D> dreader = new DataFileReader<D>(in, reader, header); // seek/sync to an (assumed) valid position if (sync) dreader.sync(in.tell()); else dreader.seek(in.tell()); return dreader; }
@Override public void reopen(FileInputSplit split, Tuple2<Long, Long> state) throws IOException { Preconditions.checkNotNull(split, "reopen() cannot be called on a null split."); Preconditions.checkNotNull(state, "reopen() cannot be called with a null initial state."); try { this.open(split); } finally { if (state.f0 != -1) { lastSync = state.f0; recordsReadSinceLastSync = state.f1; } } if (lastSync != -1) { // open and read until the record we were before // the checkpoint and discard the values dataFileReader.seek(lastSync); for(int i = 0; i < recordsReadSinceLastSync; i++) { dataFileReader.next(null); } } } }
@Override public void reopen(FileInputSplit split, Tuple2<Long, Long> state) throws IOException { Preconditions.checkNotNull(split, "reopen() cannot be called on a null split."); Preconditions.checkNotNull(state, "reopen() cannot be called with a null initial state."); try { this.open(split); } finally { if (state.f0 != -1) { lastSync = state.f0; recordsReadSinceLastSync = state.f1; } } if (lastSync != -1) { // open and read until the record we were before // the checkpoint and discard the values dataFileReader.seek(lastSync); for(int i = 0; i < recordsReadSinceLastSync; i++) { dataFileReader.next(null); } } } }
@Override public void reopen(FileInputSplit split, Tuple2<Long, Long> state) throws IOException { Preconditions.checkNotNull(split, "reopen() cannot be called on a null split."); Preconditions.checkNotNull(state, "reopen() cannot be called with a null initial state."); try { this.open(split); } finally { if (state.f0 != -1) { lastSync = state.f0; recordsReadSinceLastSync = state.f1; } } if (lastSync != -1) { // open and read until the record we were before // the checkpoint and discard the values dataFileReader.seek(lastSync); for (int i = 0; i < recordsReadSinceLastSync; i++) { dataFileReader.next(null); } } } }
@Override public void seek(String singleFileOffset) { try { // See comments for AvroFileCheckpoint to understand the behavior below AvroFileCheckpoint checkpoint = new AvroFileCheckpoint(singleFileOffset); if (checkpoint.isStartingOffset()) { // seek to the beginning of the first block fileReader.sync(0); curBlockStart = fileReader.previousSync(); curRecordOffset = 0; return; } fileReader.seek(checkpoint.getBlockStart()); for (int i = 0; i < checkpoint.getRecordOffset(); i++) { if (fileReader.hasNext()) { fileReader.next(); } } curBlockStart = checkpoint.getBlockStart(); curRecordOffset = checkpoint.getRecordOffset(); } catch (IOException e) { throw new SamzaException(e); } }