protected AvroRecordReader(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected AvroAsTextRecordReader(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
public AvroGenericRecordReader(JobConf job, FileSplit split, Reporter reporter) throws IOException { this.jobConf = job; Schema latest; try { latest = getSchema(job, split); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(); if(latest != null) { gdr.setExpected(latest); } if (split.getLength() == 0) { this.isEmptyInput = true; this.start = 0; this.reader = null; } else { this.isEmptyInput = false; this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr); this.reader.sync(split.getStart()); this.start = reader.tell(); } this.stop = split.getStart() + split.getLength(); this.recordReaderID = new UID(); }
@Override public void sync(long position) throws IOException { reader.sync(position); }
AvroRangeIterator(FileReader<D> reader, long start, long end) { this.reader = reader; this.end = end; try { reader.sync(start); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to find sync past position %d", start); } }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength()); LOG.info("Opening split " + split); dataFileReader = DataFileReader.openReader(in, datumReader); dataFileReader.sync(split.getStart()); }
@Override public void initialize(final InputSplit isplit, final TaskAttemptContext tc) throws IOException, InterruptedException { FileSplit fsplit = (FileSplit) isplit; start = fsplit.getStart(); end = fsplit.getStart() + fsplit.getLength(); DatumReader<GenericData.Array<Object>> datumReader = new GenericDatumReader<GenericData.Array<Object>>(schema); reader = DataFileReader.openReader( new FsInput(fsplit.getPath(), tc.getConfiguration()), datumReader); reader.sync(start); }
protected void init(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
@Override public void initialize(final InputSplit isplit, final TaskAttemptContext tc) throws IOException, InterruptedException { FileSplit fsplit = (FileSplit) isplit; start = fsplit.getStart(); end = fsplit.getStart() + fsplit.getLength(); DatumReader<GenericData.Record> datumReader = new GenericDatumReader<GenericData.Record>(schema); reader = DataFileReader.openReader( new FsInput(fsplit.getPath(), tc.getConfiguration()), datumReader); reader.sync(start); }
protected AvroRecordReader(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected AvroAsTextRecordReader(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected AvroAsJSONRecordReader(FileReader<GenericRecord> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected void init(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected void init(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected void init(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected AvroTextRecordReader(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
protected AvroTypedBytesRecordReader(FileReader<T> reader, FileSplit split) throws IOException { this.reader = reader; reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); DatumReader<E> datumReader; if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) { datumReader = new SpecificDatumReader<E>(avroValueType); } else { datumReader = new ReflectDatumReader<E>(avroValueType); } LOG.info("Opening split " + split); SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength()); dataFileReader = DataFileReader.openReader(in, datumReader); dataFileReader.sync(split.getStart()); }
@Override public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException, InterruptedException { FileSplit split = (FileSplit) genericSplit; Configuration conf = context.getConfiguration(); SeekableInput in = new FsInput(split.getPath(), conf); DatumReader<T> datumReader = AvroMode .fromConfiguration(context.getConfiguration()) .getReader(schema); this.reader = DataFileReader.openReader(in, datumReader); reader.sync(split.getStart()); // sync to start this.start = reader.tell(); this.end = split.getStart() + split.getLength(); }
private boolean checkAvroFileForLine(FileSystem fs, Path p, List<Integer> record) throws IOException { SeekableInput in = new FsInput(p, new Configuration()); DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); FileReader<GenericRecord> reader = DataFileReader.openReader(in, datumReader); reader.sync(0); while (reader.hasNext()) { if (valueMatches(reader.next(), record)) { return true; } } return false; }