/** Construct a reader for a file. */ public DataFileReader(File file, DatumReader<D> reader) throws IOException { this(new SeekableFileInput(file), reader, true); }
/** Construct a reader for a file. */ public DataFileReader(File file, DatumReader<D> reader) throws IOException { this(new SeekableFileInput(file), reader); }
/** Open a reader for a file. */ public static <D> FileReader<D> openReader(File file, DatumReader<D> reader) throws IOException { return openReader(new SeekableFileInput(file), reader); }
/** Open a reader for a file. */ public static <D> FileReader<D> openReader(File file, DatumReader<D> reader) throws IOException { SeekableFileInput input = new SeekableFileInput( file ); try { return openReader( input, reader ); } catch ( final Throwable e ) { IOUtils.closeQuietly( input ); throw e; } }
/** Open a writer appending to an existing file. */ public DataFileWriter<D> appendTo(File file) throws IOException { return appendTo(new SeekableFileInput(file), new SyncableFileOutputStream(file, true)); }
/** Open a writer appending to an existing file. */ public DataFileWriter<D> appendTo(File file) throws IOException { SeekableInput input = null; try { input = new SeekableFileInput(file); OutputStream output = new SyncableFileOutputStream(file, true); return appendTo(input, output); } finally { if (input != null) input.close(); // output does not need to be closed here. It will be closed by invoking close() of this writer. } }
if (useFile) { File f = makeFile(); SeekableFileInput in = new SeekableFileInput(f); try { writer.appendTo(in, out);
public void testReadWithHeader() throws IOException { File file = makeFile(); DataFileReader<Object> reader = new DataFileReader<>(file, new GenericDatumReader<>()); // get a header for this file DataFileStream.Header header = reader.getHeader(); // re-open to an arbitrary position near the middle, with sync == true SeekableFileInput sin = new SeekableFileInput(file); sin.seek(sin.length() / 2); reader = DataFileReader.openReader(sin, new GenericDatumReader<>(), header, true); assertNotNull("Should be able to reopen from arbitrary point", reader.next()); long validPos = reader.previousSync(); // post sync, we know of a valid sync point: re-open with seek (sync == false) sin.seek(validPos); reader = DataFileReader.openReader(sin, new GenericDatumReader<>(), header, false); assertEquals("Should not move from sync point on reopen", validPos, sin.tell()); assertNotNull("Should be able to reopen at sync point", reader.next()); }
final SeekableInput avroFileInput = new SeekableFileInput( AvroFiles.createFile(new File(mTempDir.getRoot(), "myStringfile.avro"), Schema.create(Schema.Type.STRING), "first", "second"));
secondInputRecord.setValue(2); final SeekableInput avroFileInput = new SeekableFileInput( AvroFiles.createFile(new File(mTempDir.getRoot(), "myInputFile.avro"), keyValueSchema, firstInputRecord.get(), secondInputRecord.get()));
@Test public void testNestedClass() throws IOException { File file = new File(DIR.getRoot().getPath(), "testNull.avro"); CheckList<BazRecord> check = new CheckList<>(); try(FileOutputStream fos = new FileOutputStream(file)) { Schema schema = ReflectData.get().getSchema(BazRecord.class); try (DataFileWriter<BazRecord> writer = new DataFileWriter<>(new ReflectDatumWriter<BazRecord>(schema))) { writer.create(schema, fos); // test writing to a file write(writer, new BazRecord(10), check); write(writer, new BazRecord(20), check); } } ReflectDatumReader<BazRecord> din = new ReflectDatumReader<>(); try(SeekableFileInput sin = new SeekableFileInput(file)) { try (DataFileReader<BazRecord> reader = new DataFileReader<>(sin, din)) { int count = 0; for (BazRecord datum : reader) { check.assertEquals(datum, count++); } Assert.assertEquals(count, check.size()); } } }
@Test public void testNull() throws IOException { File file = new File(DIR.getRoot().getPath(), "testNull.avro"); CheckList<BarRecord> check = new CheckList<>(); try(FileOutputStream fos = new FileOutputStream(file)) { ReflectData reflectData = ReflectData.AllowNull.get(); Schema schema = reflectData.getSchema(BarRecord.class); try(DataFileWriter<BarRecord> writer = new DataFileWriter<>(new ReflectDatumWriter<>(BarRecord.class, reflectData))) { writer.create(schema, fos); // test writing to a file write(writer, new BarRecord("One beer please"), check); // null record here, fails when using the default reflectData instance write(writer, new BarRecord(), check); write(writer, new BarRecord("Two beers please"), check); } } ReflectDatumReader<BarRecord> din = new ReflectDatumReader<>(); try(SeekableFileInput sin = new SeekableFileInput(file)) { try (DataFileReader<BarRecord> reader = new DataFileReader<>(sin, din)) { int count = 0; for (BarRecord datum : reader) { check.assertEquals(datum, count++); } Assert.assertEquals(count, check.size()); } } }
SeekableFileInput sin = new SeekableFileInput(file); try(DataFileReader<Object> reader = new DataFileReader<>(sin, din)) { int count = 0;
/** Construct a reader for a file. */ public DataFileReader(File file, DatumReader<D> reader) throws IOException { this(new SeekableFileInput(file), reader); }
/** Construct a reader for a file. */ public DataFileReader(File file, DatumReader<D> reader) throws IOException { this(new SeekableFileInput(file), reader); }
/** Construct a reader for a file. */ public DataFileReader(File file, DatumReader<D> reader) throws IOException { this(new SeekableFileInput(file), reader); }
/** Open a reader for a file. */ public static <D> FileReader<D> openReader(File file, DatumReader<D> reader) throws IOException { return openReader(new SeekableFileInput(file), reader); }
/** Open a reader for a file. */ public static <D> FileReader<D> openReader(File file, DatumReader<D> reader) throws IOException { return openReader(new SeekableFileInput(file), reader); }
/** Open a writer appending to an existing file. */ public DataFileWriter<D> appendTo(File file) throws IOException { return appendTo(new SeekableFileInput(file), new FileOutputStream(file, true)); }
/** Open a writer appending to an existing file. */ public DataFileWriter<D> appendTo(File file) throws IOException { return appendTo(new SeekableFileInput(file), new SyncableFileOutputStream(file, true)); }