/** * Initializes this KinesisDataFetcher's iterator based on the checkpointed sequence number. * @param initialCheckpoint Current checkpoint sequence number for this shard. * @param initialPositionInStream The initialPositionInStream. */ public void initialize(String initialCheckpoint, InitialPositionInStreamExtended initialPositionInStream) { LOG.info("Initializing shard " + shardId + " with " + initialCheckpoint); advanceIteratorTo(initialCheckpoint, initialPositionInStream); isInitialized = true; }
@Override public GetRecordsResult getRecords(final int maxRecords) { return dataFetcher.getRecords(maxRecords).accept(); }
/** * Advances this KinesisDataFetcher's internal iterator to be at the passed-in sequence number. * * @param sequenceNumber advance the iterator to the record at this sequence number. * @param initialPositionInStream The initialPositionInStream. */ void advanceIteratorTo(String sequenceNumber, InitialPositionInStreamExtended initialPositionInStream) { if (sequenceNumber == null) { throw new IllegalArgumentException("SequenceNumber should not be null: shardId " + shardId); } else if (sequenceNumber.equals(SentinelCheckpoint.LATEST.toString())) { nextIterator = getIterator(ShardIteratorType.LATEST.toString()); } else if (sequenceNumber.equals(SentinelCheckpoint.TRIM_HORIZON.toString())) { nextIterator = getIterator(ShardIteratorType.TRIM_HORIZON.toString()); } else if (sequenceNumber.equals(SentinelCheckpoint.AT_TIMESTAMP.toString())) { nextIterator = getIterator(initialPositionInStream.getTimestamp()); } else if (sequenceNumber.equals(SentinelCheckpoint.SHARD_END.toString())) { nextIterator = null; } else { nextIterator = getIterator(ShardIteratorType.AT_SEQUENCE_NUMBER.toString(), sequenceNumber); } if (nextIterator == null) { isShardEndReached = true; } this.lastKnownSequenceNumber = sequenceNumber; this.initialPositionInStream = initialPositionInStream; }
MetricsLevel.SUMMARY); dataFetcher.restartIterator(); } catch (SdkClientException e) { log.error("Exception thrown while fetching records from Kinesis", e);
ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.getCheckpoint(); dataFetcher.initialize(initialCheckpoint.getSequenceNumber(), streamConfig.getInitialPositionInStream()); getRecordsCache.start(); recordProcessorCheckpointer.setLargestPermittedCheckpointValue(initialCheckpoint);
backoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo), retryGetRecordsInSeconds, maxGetRecordsThreadPool,
if (dataFetcher.isShardEndReached()) { LOG.info("Reached end of shard " + shardInfo.getShardId()); return new TaskResult(null, true);
/** * Gets a new iterator from the last known sequence number i.e. the sequence number of the last record from the last * getRecords call. */ public void restartIterator() { if (StringUtils.isEmpty(lastKnownSequenceNumber) || initialPositionInStream == null) { throw new IllegalStateException("Make sure to initialize the KinesisDataFetcher before restarting the iterator."); } advanceIteratorTo(lastKnownSequenceNumber, initialPositionInStream); }
private Callable<DataFetcherResult> createRetrieverCallable(int maxRecords) { ThreadSafeMetricsDelegatingScope metricsScope = new ThreadSafeMetricsDelegatingScope(MetricsHelper.getMetricsScope()); return () -> { try { MetricsHelper.setMetricsScope(metricsScope); return dataFetcher.getRecords(maxRecords); } finally { MetricsHelper.unsetMetricsScope(); } }; }
public void initialize(ExtendedSequenceNumber initialCheckpoint, InitialPositionInStreamExtended initialPositionInStream) { LOG.info("Initializing shard " + shardId + " with " + initialCheckpoint.getSequenceNumber()); advanceIteratorTo(initialCheckpoint.getSequenceNumber(), initialPositionInStream); isInitialized = true; }
dataFetcher.advanceIteratorTo(recordProcessorCheckpointer.getLargestPermittedCheckpointValue() .getSequenceNumber(), streamConfig.getInitialPositionInStream());