public ListenableFuture<SegmentsAndMetadata> publishAndRegisterHandoff( final TransactionalSegmentPublisher publisher, final Committer committer, final Collection<String> sequenceNames ) { return ListenableFutures.transformAsync( publish(publisher, committer, sequenceNames), this::registerHandoff ); }
private void publishSegments( StreamAppenderatorDriver driver, TransactionalSegmentPublisher publisher, Supplier<Committer> committerSupplier, String sequenceName ) { final ListenableFuture<SegmentsAndMetadata> publishFuture = driver.publish( publisher, committerSupplier.get(), Collections.singletonList(sequenceName) ); pendingHandoffs.add(ListenableFutures.transformAsync(publishFuture, driver::registerHandoff)); }
driver.publish( sequenceMetadata.createPublisher(toolbox, ioConfig.isUseTransaction()), sequenceMetadata.getCommitterSupplier(stream, lastPersistedOffsets).get(),
@Test(timeout = 60_000L, expected = TimeoutException.class) public void testHandoffTimeout() throws Exception { final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>(); segmentHandoffNotifierFactory.disableHandoff(); Assert.assertNull(driver.startJob()); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); while (driver.getSegments().containsKey("dummy")) { Thread.sleep(100); } driver.registerHandoff(published).get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); }
final SegmentsAndMetadata published = driver.publish( publisher, committerSupplier.get(),
final SegmentsAndMetadata published = driver.publish( makeOkPublisher(), committerSupplier.get(),
@Test(timeout = 60_000L) public void testSimple() throws Exception { final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>(); Assert.assertNull(driver.startJob()); for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); while (driver.getSegments().containsKey("dummy")) { Thread.sleep(100); } final SegmentsAndMetadata segmentsAndMetadata = driver.registerHandoff(published) .get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), asIdentifiers(segmentsAndMetadata.getSegments()) ); Assert.assertEquals(3, segmentsAndMetadata.getCommitMetadata()); }
final SegmentsAndMetadata published = driver.publish( StreamAppenderatorDriverTest.makeOkPublisher(), committerSupplier.get(),
driver.publish( StreamAppenderatorDriverTest.makeOkPublisher(), committerSupplier.get(),
driver.publish( StreamAppenderatorDriverTest.makeOkPublisher(), committerSupplier.get(),
driver.publish( StreamAppenderatorDriverTest.makeFailingPublisher(failWithException), committerSupplier.get(),
public ListenableFuture<SegmentsAndMetadata> publishAndRegisterHandoff( final TransactionalSegmentPublisher publisher, final Committer committer, final Collection<String> sequenceNames ) { return ListenableFutures.transformAsync( publish(publisher, committer, sequenceNames), this::registerHandoff ); }
private void publishSegments( StreamAppenderatorDriver driver, TransactionalSegmentPublisher publisher, Supplier<Committer> committerSupplier, String sequenceName ) { final ListenableFuture<SegmentsAndMetadata> publishFuture = driver.publish( publisher, committerSupplier.get(), Collections.singletonList(sequenceName) ); pendingHandoffs.add(ListenableFutures.transformAsync(publishFuture, driver::registerHandoff)); }