/** * Read a single byte from the stream. */ @Override public int read() throws IOException { if (pos < buffer.length) { return buffer[pos++]; } if (!fileReader.hasNext()) { return -1; } writer.write(fileReader.next(), encoder); encoder.flush(); if (!fileReader.hasNext()) { // Write a new line after the last Avro record. output.write(System.getProperty("line.separator") .getBytes(StandardCharsets.UTF_8)); output.flush(); } pos = 0; buffer = output.toByteArray(); output.reset(); return read(); }
public boolean next(Text key, Text ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; datum = reader.next(datum); if (datum instanceof ByteBuffer) { ByteBuffer b = (ByteBuffer) datum; if (b.hasArray()) { int offset = b.arrayOffset(); int start = b.position(); int length = b.remaining(); key.set(b.array(), offset + start, offset + start + length); } else { byte[] bytes = new byte[b.remaining()]; b.duplicate().get(bytes); key.set(bytes); } } else { key.set(GenericData.get().toString(datum)); } return true; }
public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; wrapper.datum(reader.next(wrapper.datum())); return true; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext(); }
@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext(); }
@Override public boolean next(Text key, Text value) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) { return false; } datum = reader.next(datum); key.set(new String(datum.toString().getBytes("UTF-8"), "UTF-8")); return true; } }
@Override public E nextRecord(E reuseValue) throws IOException { if (!dataFileReader.hasNext()) { return null; } if (!reuseAvroValue) { reuseValue = InstantiationUtil.instantiate(avroValueType, Object.class); } reuseValue = dataFileReader.next(reuseValue); return reuseValue; } }
@Override public boolean hasNext() { try { return (reader.hasNext() && !reader.pastSync(end)); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to check range end: %d", end); } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { Preconditions.checkState(currentRecord != null); // Stop reading as soon as we hit a sync point out of our split: if (dataFileReader.hasNext() && !dataFileReader.pastSync(splitStart + splitLength)) { currentKey.set(currentKey.get() + 1); dataFileReader.next(currentRecord); return true; } else { return false; } }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); return true; }
public static <T extends SpecificRecordBase> T deserializeAvroMetadata(byte[] bytes, Class<T> clazz) throws IOException { DatumReader<T> reader = new SpecificDatumReader<>(clazz); FileReader<T> fileReader = DataFileReader.openReader(new SeekableByteArrayInput(bytes), reader); Preconditions .checkArgument(fileReader.hasNext(), "Could not deserialize metadata of type " + clazz); return fileReader.next(); } }
@Override public Record nextRecord(Record record) throws IOException { if (!dataFileReader.hasNext()) { return null; } reuseAvroValue = dataFileReader.next(reuseAvroValue); wrapper.datum(reuseAvroValue); record.setField(0, wrapper); return record; } }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) return false; GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)) return false; wrapper.datum(reader.next(wrapper.datum())); return true; }
public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; wrapper.datum(reader.next(wrapper.datum())); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)){ return false; } wrapper.datum(reader.next(wrapper.datum())); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)){ return false; } wrapper.datum(reader.next(wrapper.datum())); return true; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
private boolean checkAvroFileForLine(FileSystem fs, Path p, List<Integer> record) throws IOException { SeekableInput in = new FsInput(p, new Configuration()); DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); FileReader<GenericRecord> reader = DataFileReader.openReader(in, datumReader); reader.sync(0); while (reader.hasNext()) { if (valueMatches(reader.next(), record)) { return true; } } return false; }