@Override public void write(final D record) throws IOException { write(record, Ackable.NoopAckable); }
public AsyncWriterManager build() { return new AsyncWriterManager(this.config, this.commitTimeoutMillis, this.commitStepWaitTimeMillis, this.failureAllowanceRatio, this.retriesEnabled, this.numRetries, MIN_RETRY_INTERVAL_MILLIS_DEFAULT, // TODO: Make this configurable this.maxOutstandingWrites, this.asyncDataWriter, this.logger); } }
private void write(final D record, Ackable ackable) throws IOException { maybeThrow(); int spinNum = 0; try { while (!this.writePermits.tryAcquire(100, TimeUnit.MILLISECONDS)) { ++spinNum; if (spinNum % 50 == 0) { log.info("Spinning due to pending writes, in = " + this.recordsIn.getCount() + ", success = " + this.recordsSuccess.getCount() + ", failed = " + this.recordsFailed.getCount() + ", maxOutstandingWrites = " + this.maxOutstandingWrites); } } } catch (InterruptedException e) { Throwables.propagate(e); } this.recordsIn.mark(); attemptWrite(new Attempt(record, ackable)); }
@Override public DataWriter<D> build() throws IOException { validate(); return AsyncWriterManager.builder() .config(ConfigUtils.propertiesToConfig(getState().getProperties())) .asyncDataWriter(new AsyncHttpWriter(this)) .maxOutstandingWrites(maxOutstandingWrites) .retriesEnabled(false) // retries are done in HttpBatchDispatcher .commitTimeoutMillis(10000L) .failureAllowanceRatio(0).build(); } }
@Test (enabled=false) public void testCompleteFailureMode() throws Exception { FlakyAsyncWriter flakyAsyncWriter = new FlakyAsyncWriter( org.apache.gobblin.test.ErrorManager.builder().errorType(org.apache.gobblin.test.ErrorManager.ErrorType.ALL).build()); AsyncWriterManager asyncWriterManager = AsyncWriterManager.builder().asyncDataWriter(flakyAsyncWriter).retriesEnabled(true).numRetries(5).build(); byte[] messageBytes = TestUtils.generateRandomBytes(); asyncWriterManager.write(messageBytes); try { asyncWriterManager.commit(); } catch (IOException e) { // ok for commit to throw exception } finally { asyncWriterManager.close(); } Assert.assertEquals(asyncWriterManager.recordsIn.getCount(), 1); Assert.assertEquals(asyncWriterManager.recordsAttempted.getCount(), 6); Assert.assertEquals(asyncWriterManager.recordsSuccess.getCount(), 0); Assert.assertEquals(asyncWriterManager.recordsWritten(), 0); Assert.assertEquals(asyncWriterManager.recordsFailed.getCount(), 1); }
AsyncWriterManager.builder().asyncDataWriter(writer).maxOutstandingWrites(100000).retriesEnabled(true) .numRetries(5).build(); AbstractDocument doc = recordIterator.next(); verifier.onWrite(doc); asyncWriterManager.write(doc); asyncWriterManager.commit(); verifier.verify(writer.getBucket()); } finally {
AsyncWriterManager.builder().asyncDataWriter(flakyAsyncWriter).retriesEnabled(true).numRetries(5) .maxOutstandingWrites(maxOutstandingWrites).failureAllowanceRatio(1.0) // ok to fail all the time .build(); asyncWriterManager.commit(); long recordsIn = asyncWriterManager.recordsIn.getCount(); long recordsAttempted = asyncWriterManager.recordsAttempted.getCount();
/** * Generates metrics for the instrumentation of this class. */ protected void regenerateMetrics() { // Set up the metrics that are enabled regardless of instrumentation this.recordsIn = this.metricContext.meter(MetricNames.DataWriterMetrics.RECORDS_IN_METER); this.recordsAttempted = this.metricContext.meter(MetricNames.DataWriterMetrics.RECORDS_ATTEMPTED_METER); this.recordsSuccess = this.metricContext.meter(MetricNames.DataWriterMetrics.SUCCESSFUL_WRITES_METER); this.recordsFailed = this.metricContext.meter(MetricNames.DataWriterMetrics.FAILED_WRITES_METER); this.bytesWritten = this.metricContext.meter(MetricNames.DataWriterMetrics.BYTES_WRITTEN_METER); if (isInstrumentationEnabled()) { this.dataWriterTimer = Optional.<Timer>of(this.metricContext.timer(MetricNames.DataWriterMetrics.WRITE_TIMER)); } else { this.dataWriterTimer = Optional.absent(); } }
return AsyncWriterManager.builder() .failureAllowanceRatio(failureAllowance) .retriesEnabled(retriesEnabled)
AsyncWriterManager asyncWriter = AsyncWriterManager.builder().config(ConfigFactory.empty()) .commitTimeoutMillis(commitTimeoutInMillis).failureAllowanceRatio(failurePercentage / 100.0) .asyncDataWriter(fakeTimedAsyncWriter).build(); asyncWriter.write(TestUtils.generateRandomBytes()); asyncWriter.commit(); if (!success) { Assert.fail("Commit should not succeed"); asyncWriter.close(); } catch (Exception e) { Assert.fail("Should not throw any exception on close");
/** * Generates metrics for the instrumentation of this class. */ protected void regenerateMetrics() { // Set up the metrics that are enabled regardless of instrumentation this.recordsIn = this.metricContext.meter(MetricNames.DataWriterMetrics.RECORDS_IN_METER); this.recordsAttempted = this.metricContext.meter(MetricNames.DataWriterMetrics.RECORDS_ATTEMPTED_METER); this.recordsSuccess = this.metricContext.meter(MetricNames.DataWriterMetrics.SUCCESSFUL_WRITES_METER); this.recordsFailed = this.metricContext.meter(MetricNames.DataWriterMetrics.FAILED_WRITES_METER); this.bytesWritten = this.metricContext.meter(MetricNames.DataWriterMetrics.BYTES_WRITTEN_METER); if (isInstrumentationEnabled()) { this.dataWriterTimer = Optional.<Timer>of(this.metricContext.timer(MetricNames.DataWriterMetrics.WRITE_TIMER)); } else { this.dataWriterTimer = Optional.absent(); } }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_CONFIG, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_CONFIG, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_CONFIG, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)).maxOutstandingWrites(10000) .build(); } }
@Test (enabled=false) public void testFlakyWritersWithRetries() throws Exception { FlakyAsyncWriter flakyAsyncWriter = new FlakyAsyncWriter( org.apache.gobblin.test.ErrorManager.builder().errorType(ErrorManager.ErrorType.NTH).errorEvery(4).build()); AsyncWriterManager asyncWriterManager = AsyncWriterManager.builder().asyncDataWriter(flakyAsyncWriter).retriesEnabled(true).numRetries(5).build(); for (int i = 0; i < 100; ++i) { byte[] messageBytes = TestUtils.generateRandomBytes(); asyncWriterManager.write(messageBytes); } try { asyncWriterManager.commit(); } catch (IOException e) { // ok for commit to throw exception } finally { asyncWriterManager.close(); } log.info(asyncWriterManager.recordsAttempted.getCount() + ""); Assert.assertEquals(asyncWriterManager.recordsIn.getCount(), 100); Assert.assertTrue(asyncWriterManager.recordsAttempted.getCount() > 100); Assert.assertEquals(asyncWriterManager.recordsSuccess.getCount(), 100); Assert.assertEquals(asyncWriterManager.recordsFailed.getCount(), 0); }
@Override public void writeEnvelope(RecordEnvelope<D> recordEnvelope) throws IOException { write(recordEnvelope.getRecord(), recordEnvelope); }
private void write(final D record, Ackable ackable) throws IOException { maybeThrow(); int spinNum = 0; try { while (!this.writePermits.tryAcquire(100, TimeUnit.MILLISECONDS)) { ++spinNum; if (spinNum % 50 == 0) { log.info("Spinning due to pending writes, in = " + this.recordsIn.getCount() + ", success = " + this.recordsSuccess.getCount() + ", failed = " + this.recordsFailed.getCount() + ", maxOutstandingWrites = " + this.maxOutstandingWrites); } } } catch (InterruptedException e) { Throwables.propagate(e); } this.recordsIn.mark(); attemptWrite(new Attempt(record, ackable)); }
public AsyncWriterManager build() { return new AsyncWriterManager(this.config, this.commitTimeoutMillis, this.commitStepWaitTimeMillis, this.failureAllowanceRatio, this.retriesEnabled, this.numRetries, MIN_RETRY_INTERVAL_MILLIS_DEFAULT, // TODO: Make this configurable this.maxOutstandingWrites, this.asyncDataWriter, this.logger); } }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); CouchbaseEnvironment couchbaseEnvironment = CouchbaseEnvironmentFactory.getInstance(config); //TODO: Read config to decide whether to build a blocking writer or an async writer double failureAllowance = ConfigUtils.getDouble(config, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; boolean retriesEnabled = ConfigUtils.getBoolean(config, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED_DEFAULT); int maxRetries = ConfigUtils.getInt(config, CouchbaseWriterConfigurationKeys.MAX_RETRIES, CouchbaseWriterConfigurationKeys.MAX_RETRIES_DEFAULT); // build an async couchbase writer AsyncDataWriter couchbaseWriter = new CouchbaseWriter(couchbaseEnvironment, config); return AsyncWriterManager.builder() .asyncDataWriter(couchbaseWriter) .failureAllowanceRatio(failureAllowance) .retriesEnabled(retriesEnabled) .numRetries(maxRetries) .config(config) .build(); } }
@Override public void run() { GenericRecord record = TestUtils.generateRandomAvroRecord(); try { asyncWriterManager.write(record); } catch (IOException e) { log.error("Failure during write", e); Throwables.propagate(e); } } }, 0, tickDiffInNanos, TimeUnit.NANOSECONDS);
/** * Build a {@link DataWriter}. * * @throws IOException if there is anything wrong building the writer * @return the built {@link DataWriter} */ @Override public DataWriter<D> build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, KafkaWriterConfigurationKeys.COMMIT_TIMEOUT_MILLIS_CONFIG, KafkaWriterConfigurationKeys.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, KafkaWriterConfigurationKeys.COMMIT_STEP_WAIT_TIME_CONFIG, KafkaWriterConfigurationKeys.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, KafkaWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG, KafkaWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)) .build(); } }
@Override public void write(final D record) throws IOException { write(record, Ackable.NoopAckable); }