How to use

getSegments

method

in

org.apache.druid.segment.realtime.appenderator.SegmentsAndMetadata

@Override
public void onSuccess(SegmentsAndMetadata result)
{
 // Immediately publish after pushing
 for (DataSegment pushedSegment : result.getSegments()) {
  try {
   segmentPublisher.publishSegment(pushedSegment);
  }
  catch (Exception e) {
   errorHandler.apply(e);
  }
 }
 log.info("Published [%,d] sinks.", segmentsToPush.size());
}

(Function<SegmentsAndMetadata, SegmentsAndMetadata>) segmentsAndMetadata -> {
 final Set<SegmentIdWithShardSpec> pushedSegments = segmentsAndMetadata.getSegments().stream()
                                    .map(
                                      SegmentIdWithShardSpec::fromDataSegment)
 if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) {
  log.warn(
    "Removing segments from deep storage because sanity check failed: %s", segmentsAndMetadata.getSegments()
  );
  segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);

/**
 * Drop segments in background. The segments should be pushed (in batch ingestion) or published (in streaming
 * ingestion) before being dropped.
 *
 * @param segmentsAndMetadata result of pushing or publishing
 *
 * @return a future for dropping segments
 */
ListenableFuture<SegmentsAndMetadata> dropInBackground(SegmentsAndMetadata segmentsAndMetadata)
{
 log.info("Dropping segments[%s]", segmentsAndMetadata.getSegments());
 final ListenableFuture<?> dropFuture = Futures.allAsList(
   segmentsAndMetadata
     .getSegments()
     .stream()
     .map(segment -> appenderator.drop(SegmentIdWithShardSpec.fromDataSegment(segment)))
     .collect(Collectors.toList())
 );
 return Futures.transform(
   dropFuture,
   (Function<Object, SegmentsAndMetadata>) x -> {
    final Object metadata = segmentsAndMetadata.getCommitMetadata();
    return new SegmentsAndMetadata(
      segmentsAndMetadata.getSegments(),
      metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata()
    );
   }
 );
}

if (segmentsAndMetadata.getSegments().isEmpty()) {
 log.info("Nothing to publish, skipping publish step.");
} else {
   "Publishing segments with commitMetadata[%s]: [%s]",
   segmentsAndMetadata.getCommitMetadata(),
   Joiner.on(", ").join(segmentsAndMetadata.getSegments())
 );
  final Object metadata = segmentsAndMetadata.getCommitMetadata();
  final boolean published = publisher.publishSegments(
    ImmutableSet.copyOf(segmentsAndMetadata.getSegments()),
    metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata()
  ).isSuccess();
   segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);
     .getSegments()
     .stream()
     .map(SegmentIdWithShardSpec::fromDataSegment)
              .equals(Sets.newHashSet(segmentsAndMetadata.getSegments()))) {
    log.info("Our segments really do exist, awaiting handoff.");
   } else {
  log.warn(e, "Failed publish, not removing segments: %s", segmentsAndMetadata.getSegments());
  throw Throwables.propagate(e);

.getSegments()
.stream()
.collect(Collectors.toMap(SegmentIdWithShardSpec::fromDataSegment, Function.identity()));

final List<SegmentIdWithShardSpec> waitingSegmentIdList = segmentsAndMetadata.getSegments().stream()
                                       .map(
                                         SegmentIdWithShardSpec::fromDataSegment)
 return Futures.immediateFuture(
   new SegmentsAndMetadata(
     segmentsAndMetadata.getSegments(),
     ((AppenderatorDriverMetadata) metadata).getCallerMetadata()

    pushedSegments.addAll(pushed.getSegments());
    log.info("Pushed segments[%s]", pushed.getSegments());
pushedSegments.addAll(pushed.getSegments());
log.info("Pushed segments[%s]", pushed.getSegments());

log.info(
  "Handoff completed for segments %s with metadata[%s].",
  Lists.transform(handedOff.getSegments(), DataSegment::getId),
  Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata")
);

).get();
List<?> publishedSegmentIds = Lists.transform(published.getSegments(), DataSegment::getId);
log.info(
  "Published segments %s with metadata[%s].",
 log.info(
   "Handoff completed for segments %s with metadata[%s]",
   Lists.transform(handedOff.getSegments(), DataSegment::getId),
   Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata")
 );

Assert.assertEquals(numSegments, segmentsAndMetadata.getSegments().size());
Assert.assertEquals(numSegments * MAX_ROWS_PER_SEGMENT, segmentsAndMetadata.getCommitMetadata());

    log.info("Pushed segments[%s]", pushed.getSegments());
log.info("Pushed segments[%s]", pushed.getSegments());
   buildSegmentsMeters.getThrownAway()
 );
 log.info("Published segments: %s", Lists.transform(published.getSegments(), DataSegment::getId));

@Test(timeout = 60_000L)
public void testSimple() throws Exception
{
 final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>();
 Assert.assertNull(driver.startJob());
 for (int i = 0; i < ROWS.size(); i++) {
  committerSupplier.setMetadata(i + 1);
  Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk());
 }
 final SegmentsAndMetadata published = driver.publish(
   makeOkPublisher(),
   committerSupplier.get(),
   ImmutableList.of("dummy")
 ).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
 while (driver.getSegments().containsKey("dummy")) {
  Thread.sleep(100);
 }
 final SegmentsAndMetadata segmentsAndMetadata = driver.registerHandoff(published)
                            .get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
 Assert.assertEquals(
   ImmutableSet.of(
     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)),
     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0))
   ),
   asIdentifiers(segmentsAndMetadata.getSegments())
 );
 Assert.assertEquals(3, segmentsAndMetadata.getCommitMetadata());
}

@Test
public void testSimple() throws Exception
{
 Assert.assertNull(driver.startJob());
 for (InputRow row : ROWS) {
  Assert.assertTrue(driver.add(row, "dummy").isOk());
 }
 checkSegmentStates(2, SegmentState.APPENDING);
 driver.pushAllAndClear(TIMEOUT);
 checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED);
 final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get(
   TIMEOUT,
   TimeUnit.MILLISECONDS
 );
 Assert.assertEquals(
   ImmutableSet.of(
     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)),
     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0))
   ),
   published.getSegments()
        .stream()
        .map(SegmentIdWithShardSpec::fromDataSegment)
        .collect(Collectors.toSet())
 );
 Assert.assertNull(published.getCommitMetadata());
}

  sorted(
    Lists.transform(
      segmentsAndMetadata.getSegments(),
      new Function<DataSegment, SegmentIdWithShardSpec>()
Assert.assertEquals(sorted(tester.getPushedSegments()), sorted(segmentsAndMetadata.getSegments()));

     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0))
   ),
   asIdentifiers(segmentsAndMetadata.getSegments())
 );
   asIdentifiers(segmentsAndMetadata.getSegments())
 );
  asIdentifiers(segmentsAndMetadata.getSegments())
);

    new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0))
  ),
  asIdentifiers(handedoffFromSequence0.getSegments())
);
    new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0))
  ),
  asIdentifiers(handedoffFromSequence1.getSegments())
);

@Test
public void testIncrementalPush() throws Exception
{
 Assert.assertNull(driver.startJob());
 int i = 0;
 for (InputRow row : ROWS) {
  Assert.assertTrue(driver.add(row, "dummy").isOk());
  checkSegmentStates(1, SegmentState.APPENDING);
  checkSegmentStates(i, SegmentState.PUSHED_AND_DROPPED);
  driver.pushAllAndClear(TIMEOUT);
  checkSegmentStates(0, SegmentState.APPENDING);
  checkSegmentStates(++i, SegmentState.PUSHED_AND_DROPPED);
 }
 final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get(
   TIMEOUT,
   TimeUnit.MILLISECONDS
 );
 Assert.assertEquals(
   ImmutableSet.of(
     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)),
     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)),
     new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(1, 0))
   ),
   published.getSegments()
        .stream()
        .map(SegmentIdWithShardSpec::fromDataSegment)
        .collect(Collectors.toSet())
 );
 Assert.assertNull(published.getCommitMetadata());
}

@Override
public void onSuccess(SegmentsAndMetadata result)
{
 // Immediately publish after pushing
 for (DataSegment pushedSegment : result.getSegments()) {
  try {
   segmentPublisher.publishSegment(pushedSegment);
  }
  catch (Exception e) {
   errorHandler.apply(e);
  }
 }
 log.info("Published [%,d] sinks.", segmentsToPush.size());
}

(Function<SegmentsAndMetadata, SegmentsAndMetadata>) segmentsAndMetadata -> {
 final Set<SegmentIdentifier> pushedSegments = segmentsAndMetadata.getSegments().stream()
                                  .map(SegmentIdentifier::fromDataSegment)
                                  .collect(Collectors.toSet());
 if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) {
  log.warn(
    "Removing segments from deep storage because sanity check failed: %s", segmentsAndMetadata.getSegments()
  );
  segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly);

/**
 * Drop segments in background. The segments should be pushed (in batch ingestion) or published (in streaming
 * ingestion) before being dropped.
 *
 * @param segmentsAndMetadata result of pushing or publishing
 *
 * @return a future for dropping segments
 */
ListenableFuture<SegmentsAndMetadata> dropInBackground(SegmentsAndMetadata segmentsAndMetadata)
{
 log.info("Dropping segments[%s]", segmentsAndMetadata.getSegments());
 final ListenableFuture<?> dropFuture = Futures.allAsList(
   segmentsAndMetadata
     .getSegments()
     .stream()
     .map(segment -> appenderator.drop(SegmentIdentifier.fromDataSegment(segment)))
     .collect(Collectors.toList())
 );
 return Futures.transform(
   dropFuture,
   (Function<Object, SegmentsAndMetadata>) x -> {
    final Object metadata = segmentsAndMetadata.getCommitMetadata();
    return new SegmentsAndMetadata(
      segmentsAndMetadata.getSegments(),
      metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata()
    );
   }
 );
}