public boolean next(Text key, Text ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; datum = reader.next(datum); if (datum instanceof ByteBuffer) { ByteBuffer b = (ByteBuffer) datum; if (b.hasArray()) { int offset = b.arrayOffset(); int start = b.position(); int length = b.remaining(); key.set(b.array(), offset + start, offset + start + length); } else { byte[] bytes = new byte[b.remaining()]; b.duplicate().get(bytes); key.set(bytes); } } else { key.set(GenericData.get().toString(datum)); } return true; }
/** * Read a single byte from the stream. */ @Override public int read() throws IOException { if (pos < buffer.length) { return buffer[pos++]; } if (!fileReader.hasNext()) { return -1; } writer.write(fileReader.next(), encoder); encoder.flush(); if (!fileReader.hasNext()) { // Write a new line after the last Avro record. output.write(System.getProperty("line.separator") .getBytes(StandardCharsets.UTF_8)); output.flush(); } pos = 0; buffer = output.toByteArray(); output.reset(); return read(); }
public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; wrapper.datum(reader.next(wrapper.datum())); return true; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
dataFileReader.next(rec);
User rec = dataFileReader.next();
@Test public void testSerialization() throws Exception { Schema testSchema = getTestSchema(); GenericRecord message = new Record(testSchema); message.put("name", "testValue"); byte[] data = getSerializedMessage(message, testSchema); GenericDatumReader<IndexedRecord> reader = new GenericDatumReader<>(testSchema); SeekableInput in = new SeekableByteArrayInput(data); FileReader<IndexedRecord> dfr = null; IndexedRecord result = null; try { dfr = DataFileReader.openReader(in, reader); result = dfr.next(); } finally { if (dfr != null) { dfr.close(); } } Assert.assertNotNull(result); Assert.assertTrue(result instanceof GenericRecord); Assert.assertEquals(new Utf8("testValue"), ((GenericRecord) result).get("name")); } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (reader.pastSync(end)) { return false; } try { currentArray = reader.next(); } catch (NoSuchElementException e) { return false; } return true; }
@Override public D next() { if (!hasNext()) { throw new NoSuchElementException(); } try { this.reused = reader.next(reused); return reused; } catch (IOException e) { throw new RuntimeIOException(e, "Failed to read next record"); } }
@Override public D next(D reuse) { if (!hasNext()) { throw new NoSuchElementException(); } try { return reader.next(reuse); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to read next record"); } }
public boolean next(Text key, Text value) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) { return false; } StringBuilder buf = new StringBuilder(); JSONUtils.writeJSON(reader.next(), buf); key.set(buf.toString()); return true; }
@Override public E nextRecord(E reuseValue) throws IOException { if (!dataFileReader.hasNext()) { return null; } if (!reuseAvroValue) { reuseValue = InstantiationUtil.instantiate(avroValueType, Object.class); } reuseValue = dataFileReader.next(reuseValue); return reuseValue; } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { Preconditions.checkState(currentRecord != null); // Stop reading as soon as we hit a sync point out of our split: if (dataFileReader.hasNext() && !dataFileReader.pastSync(splitStart + splitLength)) { currentKey.set(currentKey.get() + 1); dataFileReader.next(currentRecord); return true; } else { return false; } }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); return true; }
public static <T extends SpecificRecordBase> T deserializeAvroMetadata(byte[] bytes, Class<T> clazz) throws IOException { DatumReader<T> reader = new SpecificDatumReader<>(clazz); FileReader<T> fileReader = DataFileReader.openReader(new SeekableByteArrayInput(bytes), reader); Preconditions .checkArgument(fileReader.hasNext(), "Could not deserialize metadata of type " + clazz); return fileReader.next(); } }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) return false; GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)) return false; wrapper.datum(reader.next(wrapper.datum())); return true; }
public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; wrapper.datum(reader.next(wrapper.datum())); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)){ return false; } wrapper.datum(reader.next(wrapper.datum())); return true; }
private boolean checkAvroFileForLine(FileSystem fs, Path p, List<Integer> record) throws IOException { SeekableInput in = new FsInput(p, new Configuration()); DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); FileReader<GenericRecord> reader = DataFileReader.openReader(in, datumReader); reader.sync(0); while (reader.hasNext()) { if (valueMatches(reader.next(), record)) { return true; } } return false; }