@Override public void handle(Event event, Throwable cause) throws EventDeliveryException { try { if (writer == null) { writer = dataset.newWriter(); } final AvroFlumeEvent avroEvent = new AvroFlumeEvent(); avroEvent.setBody(ByteBuffer.wrap(event.getBody())); avroEvent.setHeaders(toCharSeqMap(event.getHeaders())); writer.write(avroEvent); nEventsHandled++; } catch (RuntimeException ex) { throw new EventDeliveryException(ex); } }
@Override public void close() throws EventDeliveryException { if (nEventsHandled > 0) { try { writer.close(); } catch (RuntimeException ex) { throw new EventDeliveryException(ex); } finally { writer = null; nEventsHandled = 0; } } }
private void populateInputDataset() { DatasetWriter<GenericData.Record> writer = inputDataset.newWriter(); writer.write(newStringRecord("apple")); writer.write(newStringRecord("banana")); writer.write(newStringRecord("banana")); writer.write(newStringRecord("carrot")); writer.write(newStringRecord("apple")); writer.write(newStringRecord("apple")); writer.close(); }
/** * Parse the event using the entity parser and write the entity to the dataset. * * @param event The event to write * @throws EventDeliveryException An error occurred trying to write to the dataset that couldn't or shouldn't be handled by the failure policy. */ @VisibleForTesting void write(Event event) throws EventDeliveryException { try { this.entity = parser.parse(event, reuseEntity ? entity : null); this.bytesParsed += event.getBody().length; // writeEncoded would be an optimization in some cases, but HBase // will not support it and partitioned Datasets need to get partition // info from the entity Object. We may be able to avoid the // serialization round-trip otherwise. writer.write(entity); } catch (NonRecoverableEventException ex) { failurePolicy.handle(event, ex); } catch (DataFileWriter.AppendWriteException ex) { failurePolicy.handle(event, ex); } catch (RuntimeException ex) { Throwables.propagateIfInstanceOf(ex, EventDeliveryException.class); throw new EventDeliveryException(ex); } }
if (writer != null) { try { writer.close();
@Override public void process(InputStream in) throws IOException { try (DataFileStream<Record> stream = new DataFileStream<>( in, AvroUtil.newDatumReader(schema, Record.class))) { IncompatibleSchemaException.check( SchemaValidationUtil.canRead(stream.getSchema(), schema), "Incompatible file schema %s, expected %s", stream.getSchema(), schema); long written = 0L; try (DatasetWriter<Record> writer = target.newWriter()) { for (Record record : stream) { writer.write(record); written += 1; } } finally { session.adjustCounter("Stored records", written, true /* cannot roll back the write */); } } } });
@Override public void close(TaskAttemptContext taskAttemptContext) { datasetWriter.close(); } }
@Override public int run(List<String> args) throws Exception { Preconditions.checkState(!Datasets.exists(uri), "events dataset already exists"); DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schema(StandardEvent.class).build(); View<StandardEvent> events = Datasets.create(uri, descriptor, StandardEvent.class); DatasetWriter<StandardEvent> writer = events.newWriter(); try { while (System.currentTimeMillis() - baseTimestamp < 36000) { writer.write(generateRandomEvent()); } } finally { writer.close(); } System.out.println("Generated " + counter + " events"); return 0; }
@Override public void write(E key, Void v) { if (copyRecords) { key = copy(key); } datasetWriter.write(key); }
@Override public void close(TaskAttemptContext taskAttemptContext) { datasetWriter.close(); } }
private static void writeTestRecords(View<TestRecord> view) { DatasetWriter<TestRecord> writer = null; try { writer = view.newWriter(); for (int i = 0; i < 10; i += 1) { TestRecord record = new TestRecord(); record.id = i; record.data = "test/-" + i; writer.write(record); } } finally { if (writer != null) { writer.close(); } } } }
@Override public void write(E key, Void v) { if (copyRecords) { key = copy(key); } datasetWriter.write(key); }
df.toJavaRDD().foreachPartition(new VoidFunction<Iterator<Row>>() { @Override public void call(Iterator<Row> iterator) throws Exception { final DatasetWriter writer = // obtain writer while (iterator.hasNext()) { // process the records; write to HDFS using writer } writer.close(); // this ensures that .avro.tmp is converted to .avro } });
private static void writeTestRecords(View<TestRecord> view) { DatasetWriter<TestRecord> writer = null; try { writer = view.newWriter(); for (int i = 0; i < 10; i += 1) { TestRecord record = new TestRecord(); record.id = i; record.data = "test-" + i; writer.write(record); } } finally { if (writer != null) { writer.close(); } } } }
@Override public Void call() throws IOException { DatasetWriter<StandardEvent> writer = null; try { writer = range.newWriter(); writer.write(event); } finally { Closeables.close(writer, false); } return null; } });
@Override public void close() throws IOException { if (log.isDebugEnabled()) { log.debug("Closing writer " + writer); } if (writer != null) { writer.close(); writer = null; } }