final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob();
@Test public void testSimple() throws Exception { Assert.assertNull(driver.startJob()); for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); } checkSegmentStates(2, SegmentState.APPENDING); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED); final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob();
@Test public void testIncrementalPush() throws Exception { Assert.assertNull(driver.startJob()); int i = 0; for (InputRow row : ROWS) { Assert.assertTrue(driver.add(row, "dummy").isOk()); checkSegmentStates(1, SegmentState.APPENDING); checkSegmentStates(i, SegmentState.PUSHED_AND_DROPPED); driver.pushAllAndClear(TIMEOUT); checkSegmentStates(0, SegmentState.APPENDING); checkSegmentStates(++i, SegmentState.PUSHED_AND_DROPPED); } final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( TIMEOUT, TimeUnit.MILLISECONDS ); Assert.assertEquals( ImmutableSet.of( new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(1, 0)) ), published.getSegments() .stream() .map(SegmentIdWithShardSpec::fromDataSegment) .collect(Collectors.toSet()) ); Assert.assertNull(published.getCommitMetadata()); }
final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob();
final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob();