@Override public void writeImpl(D record) throws IOException { this.embeddedWriter.write(record); }
@Override public void commit() throws IOException { writer.commit(); }
@Override public long bytesWritten() throws IOException { return writer.bytesWritten(); }
this.writer.write(convertRecord(record)); Assert.assertEquals(this.writer.recordsWritten(), 3); this.writer.close(); this.writer.commit();
this.writer.writeEnvelope(new RecordEnvelope<>(genericRecordBuilder.build())); this.writer.writeEnvelope(new RecordEnvelope<>(genericRecordBuilder.build())); this.writer.writeEnvelope(new RecordEnvelope<>(genericRecordBuilder.build())); Assert.assertEquals(this.writer.recordsWritten(), 3); this.writer.close(); this.writer.commit();
stream.getRecordStream().subscribe(r -> { if (r instanceof RecordEnvelope) { this.writer.get().writeEnvelope((RecordEnvelope) r); } else if (r instanceof ControlMessage) { this.writer.get().getMessageHandler().handleMessage((ControlMessage) r); r.ack(); () -> { if (this.writer.isPresent()) { this.writer.get().close();
for (int i = 0; i < RECORD_SIZE; i++) { genericRecordBuilder.set(PARTITION_COLUMN_NAME, recordTimestamps[i]); this.writer.writeEnvelope(new RecordEnvelope<>(genericRecordBuilder.build())); this.writer.close(); this.writer.commit();
@Override public long recordsWritten() { return writer.recordsWritten(); }
@Override public void cleanup() throws IOException { wrappedWriter.cleanup(); }
@Override public Void call() throws Exception { writer.writeEnvelope(recordEnvelope); return null; } };
/** * Calls inner writer with self throttling. * If the throttle type is byte, it applies throttle after write happens. * This is because it can figure out written bytes after it's written. It's not ideal but throttling after write should be sufficient for most cases. * {@inheritDoc} * @see org.apache.gobblin.writer.DataWriter#write(java.lang.Object) */ @Override public void writeEnvelope(RecordEnvelope<D> record) throws IOException { try { if (ThrottleType.QPS.equals(type)) { acquirePermits(1L); } long beforeWrittenBytes = writer.bytesWritten(); writer.writeEnvelope(record); if (ThrottleType.Bytes.equals(type)) { long delta = writer.bytesWritten() - beforeWrittenBytes; if (delta < 0) { throw new UnsupportedOperationException("Cannot throttle on bytes because " + writer.getClass().getSimpleName() + " does not supports bytesWritten"); } if (delta > 0) { acquirePermits(delta); } } } catch (InterruptedException e) { throw new IOException("Failed while acquiring permits.",e); } }
@Override public void close() throws IOException { wrappedWriter.close(); }
@Override public void flush() throws IOException { this.embeddedWriter.flush(); } }
@Override public ControlMessageHandler getMessageHandler() { return this.writer.getMessageHandler(); }
@Override public Descriptor getDataDescriptor() { return this.embeddedWriter.getDataDescriptor(); }
stream.getRecordStream().subscribe(r -> { if (r instanceof RecordEnvelope) { this.writer.get().writeEnvelope((RecordEnvelope) r); } else if (r instanceof ControlMessage) { this.writer.get().getMessageHandler().handleMessage((ControlMessage) r); r.ack(); () -> { if (this.writer.isPresent()) { this.writer.get().close();