public AvroFileInputStream(FileStatus status) throws IOException { pos = 0; buffer = new byte[0]; GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(new Configuration()); fileReader = DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader); Schema schema = fileReader.getSchema(); writer = new GenericDatumWriter<Object>(schema); output = new ByteArrayOutputStream(); encoder = EncoderFactory.get().jsonEncoder(schema, output); }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
@Override public Schema getSchema() { return reader.getSchema(); }
@Override public Schema getSchema(Configuration conf, Path path) throws IOException { SeekableInput input = new FsInput(path, conf); DatumReader<Object> reader = new GenericDatumReader<>(); FileReader<Object> fileReader = DataFileReader.openReader(input, reader); org.apache.avro.Schema schema = fileReader.getSchema(); fileReader.close(); return avroData.toConnectSchema(schema); }
public AvroFileInputStream(FileStatus status) throws IOException { pos = 0; buffer = new byte[0]; GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(new Configuration()); fileReader = DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader); Schema schema = fileReader.getSchema(); writer = new GenericDatumWriter<Object>(schema); output = new ByteArrayOutputStream(); JsonGenerator generator = new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8); MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter(); prettyPrinter.setRootValueSeparator(System.getProperty("line.separator")); generator.setPrettyPrinter(prettyPrinter); encoder = EncoderFactory.get().jsonEncoder(schema, generator); }
public AvroFileInputStream(FileStatus status) throws IOException { pos = 0; buffer = new byte[0]; GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(new Configuration()); fileReader = DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader); Schema schema = fileReader.getSchema(); writer = new GenericDatumWriter<Object>(schema); output = new ByteArrayOutputStream(); JsonGenerator generator = new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8); MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter(); prettyPrinter.setRootValueSeparator(System.getProperty("line.separator")); generator.setPrettyPrinter(prettyPrinter); encoder = EncoderFactory.get().jsonEncoder(schema, generator); }
FileReader<Object> fileReader = DataFileReader.openReader(input, reader); try { Schema schema = fileReader.getSchema(); DatumWriter<Object> writer = new GenericDatumWriter<>(schema); JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, output);
public AvroFileInputStream(FileStatus status) throws IOException { pos = 0; buffer = new byte[0]; GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(new Configuration()); fileReader = DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader); Schema schema = fileReader.getSchema(); writer = new GenericDatumWriter<Object>(schema); output = new ByteArrayOutputStream(); JsonGenerator generator = new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8); MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter(); prettyPrinter.setRootValueSeparator(System.getProperty("line.separator")); generator.setPrettyPrinter(prettyPrinter); encoder = EncoderFactory.get().jsonEncoder(schema, generator); }
schema = dataFileReader.getSchema(); currentRecord = new GenericData.Record(schema);
private boolean isSchemaCompatible(Schema inputSchema, Path filePath) throws IOException { GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(configuration); FileReader fileReader = DataFileReader.openReader(new AvroFSInput(fc, filePath), reader); Schema fileSchema = fileReader.getSchema(); if (Compatibility.checkThat(fileSchema).canRead(inputSchema).getResult().getCompatibility().equals(patched.org.apache.avro.SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE)) { return true; } return false; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
List<GenericData.Record> expecteds = new ArrayList(); FileReader<GenericData.Record> reader = new DataFileReader(file, new GenericDatumReader()); Schema schema = reader.getSchema(); while (reader.hasNext()) { GenericData.Record expected = reader.next();
List<GenericData.Record> expecteds = new ArrayList(); FileReader<GenericData.Record> reader = new DataFileReader(file, new GenericDatumReader()); Schema schema = reader.getSchema(); while (reader.hasNext()) { GenericData.Record expected = reader.next();