@Override public ITask createTask(ShardConsumer consumer) { return new InitializeTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getCheckpoint(), consumer.getRecordProcessorCheckpointer(), consumer.getDataFetcher(), consumer.getTaskBackoffTimeMillis(), consumer.getStreamConfig(), consumer.getGetRecordsCache()); }
/** * @param maxCacheByteSize Max byte size for the cache at any given point of time. After this threshold is crossed * the KinesisDataFetcher will be blocked until the cache has more space available. * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withMaxCacheByteSize(final int maxCacheByteSize) { checkIsValuePositive("maxCacheByteSize", maxCacheByteSize); this.recordsFetcherFactory.setMaxByteSize(maxCacheByteSize); return this; }
@Override public ITask createTask(ShardConsumer consumer) { return new BlockOnParentShardTask(consumer.getShardInfo(), consumer.getLeaseManager(), consumer.getParentShardPollIntervalMillis()); }
@Override public ITask createTask(ShardConsumer consumer) { return new ShutdownTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getShutdownReason(), consumer.getStreamConfig().getStreamProxy(), consumer.getStreamConfig().getInitialPositionInStream(), consumer.isCleanupLeasesOfCompletedShards(), consumer.isIgnoreUnexpectedChildShards(), consumer.getLeaseManager(), consumer.getTaskBackoffTimeMillis(), consumer.getGetRecordsCache()); }
@Override public ITask createTask(ShardConsumer consumer) { return new ProcessTask(consumer.getShardInfo(), consumer.getStreamConfig(), consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getDataFetcher(), consumer.getTaskBackoffTimeMillis(), consumer.isSkipShardSyncAtWorkerInitializationIfLeasesExist(), consumer.getGetRecordsCache()); }
@Override public ITask createTask(ShardConsumer consumer) { return new ShutdownNotificationTask(consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getShutdownNotification(), consumer.getShardInfo()); }
private void logTaskException(TaskResult taskResult) { if (LOG.isDebugEnabled()) { Exception taskException = taskResult.getException(); if (taskException instanceof BlockedOnParentShardException) { // No need to log the stack trace for this exception (it is very specific). LOG.debug("Shard " + shardInfo.getShardId() + " is blocked on completion of parent shard."); } else { LOG.debug("Caught exception running " + currentTask.getTaskType() + " task: ", taskResult.getException()); } } }
private static final GetRecordsRetrievalStrategy makeStrategy(KinesisDataFetcher dataFetcher, Optional<Integer> retryGetRecordsInSeconds, Optional<Integer> maxGetRecordsThreadPool, ShardInfo shardInfo) { Optional<GetRecordsRetrievalStrategy> getRecordsRetrievalStrategy = retryGetRecordsInSeconds.flatMap(retry -> maxGetRecordsThreadPool.map(max -> new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, retry, max, shardInfo.getShardId()))); return getRecordsRetrievalStrategy.orElse(new SynchronousGetRecordsRetrievalStrategy(dataFetcher)); }
/** * @param idleMillisBetweenCalls Idle time between 2 getcalls from the data fetcher. * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withIdleMillisBetweenCalls(long idleMillisBetweenCalls) { checkIsValuePositive("IdleMillisBetweenCalls", idleMillisBetweenCalls); this.recordsFetcherFactory.setIdleMillisBetweenCalls(idleMillisBetweenCalls); return this; }
private static ExtendedSequenceNumber convertToCheckpoint(InitialPositionInStreamExtended position) { ExtendedSequenceNumber checkpoint = null; if (position.getInitialPositionInStream().equals(InitialPositionInStream.TRIM_HORIZON)) { checkpoint = ExtendedSequenceNumber.TRIM_HORIZON; } else if (position.getInitialPositionInStream().equals(InitialPositionInStream.LATEST)) { checkpoint = ExtendedSequenceNumber.LATEST; } else if (position.getInitialPositionInStream().equals(InitialPositionInStream.AT_TIMESTAMP)) { checkpoint = ExtendedSequenceNumber.AT_TIMESTAMP; } return checkpoint; }
/** * Checks if the string is all digits or one of the SentinelCheckpoint values. * * @param string * @return true if and only if the string is all digits or one of the SentinelCheckpoint values */ private static boolean isDigitsOrSentinelValue(String string) { return SequenceNumberValidator.isDigits(string) || isSentinelValue(string); }
@Override public ITask createTask(ShardConsumer consumer) { if (consumer.getShutdownNotification() != null) { consumer.getShutdownNotification().shutdownComplete(); } return null; }
/** * * @param maxPendingProcessRecordsInput The max number of ProcessRecordsInput that can be stored in the cache before * blocking * @return this configuration object */ public KinesisClientLibConfiguration withMaxPendingProcessRecordsInput(final int maxPendingProcessRecordsInput) { checkIsValuePositive("maxPendingProcessRecordsInput", maxPendingProcessRecordsInput); this.recordsFetcherFactory.setMaxPendingProcessRecordsInput(maxPendingProcessRecordsInput); return this; }
/** * @param maxRecordsCount The maximum number of records in the cache, accross all ProcessRecordInput objects * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withMaxRecordsCount(final int maxRecordsCount) { checkIsValuePositive("maxRecordsCount", maxRecordsCount); this.recordsFetcherFactory.setMaxRecordsCount(maxRecordsCount); return this; }
/** * Figure out next task to run based on current state, task, and shutdown context. * * @return Return next task to run */ private ITask getNextTask() { ITask nextTask = currentState.createTask(this); if (nextTask == null) { return null; } else { return new MetricsCollectingTaskDecorator(nextTask, metricsFactory); } }
@Override public GetRecordsResult getRecords(final int maxRecords) { return dataFetcher.getRecords(maxRecords).accept(); }
/** * @param initialLeaseTableWriteCapacity Write capacity to provision when creating the lease table. * @return KinesisClientLibConfiguration */ public KinesisClientLibConfiguration withInitialLeaseTableWriteCapacity(int initialLeaseTableWriteCapacity) { checkIsValuePositive("initialLeaseTableWriteCapacity", initialLeaseTableWriteCapacity); this.initialLeaseTableWriteCapacity = initialLeaseTableWriteCapacity; return this; }
/** * @return The timestamp from where we need to start the application. * Valid only for initial position of type AT_TIMESTAMP, returns null for other positions. */ public Date getTimestampAtInitialPositionInStream() { return initialPositionInStreamExtended.getTimestamp(); }
@Override public void shutdown() { getRecordsRetrievalStrategy.shutdown(); } }
/** * @param listShardsBackoffTimeInMillis Max sleep between two listShards call when throttled * in {@link com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisProxy}. * @return */ public KinesisClientLibConfiguration withListShardsBackoffTimeInMillis(long listShardsBackoffTimeInMillis) { checkIsValuePositive("listShardsBackoffTimeInMillis", listShardsBackoffTimeInMillis); this.listShardsBackoffTimeInMillis = listShardsBackoffTimeInMillis; return this; }