void close() throws IOException { mAvroFileWriter.close(); }
@Override public void close() throws IOException { _recordWriter.close(); } }
@Override public void close(boolean abort) throws IOException { dfw.close(); }
/** {@inheritDoc} */ @Override public void close(TaskAttemptContext context) throws IOException { mAvroFileWriter.close(); }
@Override public void close() throws IOException { writer.close(); } }
/** {@inheritDoc} */ @Override public void close(TaskAttemptContext context) throws IOException { mAvroFileWriter.close(); }
@Override public void close() throws IOException { writer.close(); } }
public void close(Reporter reporter) throws IOException { writer.close(); } };
@Override protected void doClose() throws IOException { this.avroWriter.close(); this.out.close(); } }
private static byte[] convertRecordToAvro(Schema schema, Map<String, Object> values) { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GenericData.Record record = new GenericData.Record(schema); values.forEach(record::put); try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(new GenericDatumWriter<>(schema))) { dataFileWriter.create(schema, outputStream); dataFileWriter.append(record); dataFileWriter.close(); } catch (IOException e) { throw new UncheckedIOException("Failed to convert to Avro.", e); } return outputStream.toByteArray(); }
@Override public void close() throws IOException { if (isOpen) { writer.close(); reader.close(); isOpen = false; } }
@Override public void close() throws IOException { dataFileWriter.flush(); dataFileWriter.close(); super.close(); } }
private byte[] serializeToAvro(Schema avroSchema, List<Record> docList) throws IOException { serializationBuffer.reset(); dataFileWriter.create(avroSchema, serializationBuffer); for (Record doc2 : docList) { dataFileWriter.append(doc2); } dataFileWriter.close(); return serializationBuffer.toByteArray(); }
private static GenericData.Record buildAvroRecord(Schema schema, ByteArrayOutputStream outputStream, Map<String, Object> values) { GenericData.Record record = new GenericData.Record(schema); values.forEach(record::put); try { DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(new GenericDatumWriter<>(schema)); dataFileWriter.create(schema, outputStream); dataFileWriter.append(record); dataFileWriter.close(); } catch (IOException e) { throw new RuntimeException("Failed to convert to Avro.", e); } return record; }
private byte[] writeRecord(Schema schema, GenericData.Record record) throws Exception { ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); GenericDatumWriter<GenericData.Record> datumWriter = new GenericDatumWriter<>(schema); DataFileWriter<GenericData.Record> writer = new DataFileWriter<>(datumWriter); try { writer.create(schema, byteStream); writer.append(record); } finally { writer.close(); } return byteStream.toByteArray(); }
private ByteArrayOutputStream serializeAvroRecord(Schema schema, GenericRecord user2, DatumWriter<GenericRecord> datumWriter, Map<String, String> metadata) throws IOException { ByteArrayOutputStream out2 = new ByteArrayOutputStream(); DataFileWriter<GenericRecord> dataFileWriter2 = new DataFileWriter<>(datumWriter); if (metadata != null) { metadata.forEach(dataFileWriter2::setMeta); } dataFileWriter2.create(schema, out2); dataFileWriter2.append(user2); dataFileWriter2.close(); return out2; }
public void createAvroFileWithRepeatingRecords(File file, GenericRecord r, int count) throws IOException { DataFileWriter<GenericRecord> writer = new DataFileWriter<>(new GenericDatumWriter<GenericRecord>()); writer.create(getSchema(), new FileOutputStream(file)); for (int i = 0; i < count; ++i) { writer.append(r); } writer.close(); }
public void writeLinesFile() throws IOException { FileUtil.fullyDelete(dir); DatumWriter<String> writer = new GenericDatumWriter<>(); DataFileWriter<String> out = new DataFileWriter<>(writer); linesFiles.getParentFile().mkdirs(); out.create(Schema.create(Schema.Type.STRING), linesFiles); for (String line : LINES) out.append(line); out.close(); }
@Test() public void testUseMeta() throws IOException { DataFileWriter<?> w = new DataFileWriter<>(new GenericDatumWriter<>()); File f = new File(DIR.getRoot().getPath(), "testDataFileMeta.avro"); w.setMeta("hello", "bar"); w.create(Schema.create(Type.NULL), f); w.close(); DataFileStream<Void> r = new DataFileStream<>(new FileInputStream(f), new GenericDatumReader<>()); assertTrue(r.getMetaKeys().contains("hello")); assertEquals("bar", r.getMetaString("hello")); }