@Override public void close() throws IOException { if (isEmptyInput == false) reader.close(); }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
/** * Read a single byte from the stream. */ @Override public int read() throws IOException { if (pos < buffer.length) { return buffer[pos++]; } if (!fileReader.hasNext()) { return -1; } writer.write(fileReader.next(), encoder); encoder.flush(); if (!fileReader.hasNext()) { // Write a new line after the last Avro record. output.write(System.getProperty("line.separator") .getBytes(StandardCharsets.UTF_8)); output.flush(); } pos = 0; buffer = output.toByteArray(); output.reset(); return read(); }
@Test public void testSerialization() throws Exception { Schema testSchema = getTestSchema(); GenericRecord message = new Record(testSchema); message.put("name", "testValue"); byte[] data = getSerializedMessage(message, testSchema); GenericDatumReader<IndexedRecord> reader = new GenericDatumReader<>(testSchema); SeekableInput in = new SeekableByteArrayInput(data); FileReader<IndexedRecord> dfr = null; IndexedRecord result = null; try { dfr = DataFileReader.openReader(in, reader); result = dfr.next(); } finally { if (dfr != null) { dfr.close(); } } Assert.assertNotNull(result); Assert.assertTrue(result instanceof GenericRecord); Assert.assertEquals(new Utf8("testValue"), ((GenericRecord) result).get("name")); } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (reader.pastSync(end)) { return false; } try { currentRecord = reader.next(new GenericData.Record(schema)); } catch (NoSuchElementException e) { return false; } catch (IOException ioe) { reader.sync(reader.tell()+1); throw ioe; } return true; }
private boolean checkAvroFileForLine(FileSystem fs, Path p, List<Integer> record) throws IOException { SeekableInput in = new FsInput(p, new Configuration()); DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); FileReader<GenericRecord> reader = DataFileReader.openReader(in, datumReader); reader.sync(0); while (reader.hasNext()) { if (valueMatches(reader.next(), record)) { return true; } } return false; }
dataFileReader.next(rec);
@Override public Schema getSchema(Configuration conf, Path path) throws IOException { SeekableInput input = new FsInput(path, conf); DatumReader<Object> reader = new GenericDatumReader<>(); FileReader<Object> fileReader = DataFileReader.openReader(input, reader); org.apache.avro.Schema schema = fileReader.getSchema(); fileReader.close(); return avroData.toConnectSchema(schema); }
@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext(); }
@Override public void sync(long position) throws IOException { reader.sync(position); }
@Override public long getPos() throws IOException { return isEmptyInput ? 0 : reader.tell(); }
public AvroFileInputStream(FileStatus status) throws IOException { pos = 0; buffer = new byte[0]; GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(new Configuration()); fileReader = DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader); Schema schema = fileReader.getSchema(); writer = new GenericDatumWriter<Object>(schema); output = new ByteArrayOutputStream(); encoder = EncoderFactory.get().jsonEncoder(schema, output); }
@Override public boolean pastSync(long position) throws IOException { return reader.pastSync(position); }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }