/** * Get KCL config for a given system stream. * @param system name of the system * @param stream name of the stream * @param appName name of the application * @return Stream scoped KCL configs required to build * {@link KinesisClientLibConfiguration} */ public KinesisClientLibConfiguration getKinesisClientLibConfig(String system, String stream, String appName) { ClientConfiguration clientConfig = getAWSClientConfig(system); String workerId = appName + "-" + UUID.randomUUID(); InitialPositionInStream startPos = InitialPositionInStream.LATEST; AWSCredentialsProvider provider = credentialsProviderForStream(system, stream); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(appName, stream, provider, workerId) .withRegionName(getRegion(system, stream).getName()) .withKinesisClientConfig(clientConfig) .withCloudWatchClientConfig(clientConfig) .withDynamoDBClientConfig(clientConfig) .withInitialPositionInStream(startPos) .withCallProcessRecordsEvenForEmptyRecordList(true); // For health monitoring metrics. // First, get system scoped configs for KCL and override with configs set at stream scope. setKinesisClientLibConfigs( subset(String.format(CONFIG_SYSTEM_KINESIS_CLIENT_LIB_CONFIG, system)), kinesisClientLibConfiguration); setKinesisClientLibConfigs(subset(String.format(CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, system, stream)), kinesisClientLibConfiguration); return kinesisClientLibConfiguration; }
.withInitialPositionInStream(kinesisConnectorConfiguration.INITIAL_POSITION_IN_STREAM) .withIdleTimeBetweenReadsInMillis(kinesisConnectorConfiguration.IDLE_TIME_BETWEEN_READS) .withCallProcessRecordsEvenForEmptyRecordList(KinesisConnectorConfiguration.DEFAULT_CALL_PROCESS_RECORDS_EVEN_FOR_EMPTY_LIST) .withCleanupLeasesUponShardCompletion(kinesisConnectorConfiguration.CLEANUP_TERMINATED_SHARDS_BEFORE_EXPIRY) .withParentShardPollIntervalMillis(kinesisConnectorConfiguration.PARENT_SHARD_POLL_INTERVAL)
.withInitialPositionInStream(kinesisConnectorConfiguration.INITIAL_POSITION_IN_STREAM) .withIdleTimeBetweenReadsInMillis(kinesisConnectorConfiguration.IDLE_TIME_BETWEEN_READS) .withCallProcessRecordsEvenForEmptyRecordList(KinesisConnectorConfiguration.DEFAULT_CALL_PROCESS_RECORDS_EVEN_FOR_EMPTY_LIST) .withCleanupLeasesUponShardCompletion(kinesisConnectorConfiguration.CLEANUP_TERMINATED_SHARDS_BEFORE_EXPIRY) .withParentShardPollIntervalMillis(kinesisConnectorConfiguration.PARENT_SHARD_POLL_INTERVAL)
public void startReplicatingChanges() throws StreamNotEnabledException { if (tableSchema == null) { throw new TableExistsException("table %s does not exist in destination", dynamoTableName); } String tableStreamArn = getStreamArn(); if (tableStreamArn == null) { throw new StreamNotEnabledException("table %s does not have a stream enabled\n", dynamoTableName); } AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient); AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.standard().build(); String workerId = generateWorkerId(); final KinesisClientLibConfiguration workerConfig = new KinesisClientLibConfiguration( APPLICATION_NAME, tableStreamArn, awsCredentialsProvider, workerId). withMaxRecords(1000). withIdleTimeBetweenReadsInMillis(500). withCallProcessRecordsEvenForEmptyRecordList(false). withCleanupLeasesUponShardCompletion(false). withFailoverTimeMillis(20000). withTableName(LEASE_TABLE_PREFIX + dynamoTableName). withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); Worker worker = new Worker.Builder(). recordProcessorFactory(recordProcessorFactory). config(workerConfig). kinesisClient(adapterClient). cloudWatchClient(cloudWatchClient). dynamoDBClient(dynamoDBClient). execService(executor). build(); executor.execute(worker); }