public void start() { int mb = 1024 * 1024; LOG.info("Max memory: {} mb", Runtime.getRuntime().maxMemory() / mb); LOG.info("Starting up Kinesis Consumer... (may take a few seconds)"); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisCfg.getKinesisCredentialsProvider(), kinesisCfg.getKinesisClientConfiguration()); AmazonDynamoDBClient dynamoDBClient = new AmazonDynamoDBClient(kinesisCfg.getDynamoDBCredentialsProvider(), kinesisCfg.getDynamoDBClientConfiguration()); AmazonCloudWatch cloudWatchClient = new AmazonCloudWatchClient(kinesisCfg.getCloudWatchCredentialsProvider(), kinesisCfg.getCloudWatchClientConfiguration()); Worker worker = new Worker.Builder() .recordProcessorFactory(() -> new RecordProcessor(unitOfWorkListener, exceptionStrategy, metricsCallback, dry)) .config(kinesisCfg) .kinesisClient(kinesisClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .build(); worker.run(); }
public void startReplicatingChanges() throws StreamNotEnabledException { if (tableSchema == null) { throw new TableExistsException("table %s does not exist in destination", dynamoTableName); } String tableStreamArn = getStreamArn(); if (tableStreamArn == null) { throw new StreamNotEnabledException("table %s does not have a stream enabled\n", dynamoTableName); } AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient); AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.standard().build(); String workerId = generateWorkerId(); final KinesisClientLibConfiguration workerConfig = new KinesisClientLibConfiguration( APPLICATION_NAME, tableStreamArn, awsCredentialsProvider, workerId). withMaxRecords(1000). withIdleTimeBetweenReadsInMillis(500). withCallProcessRecordsEvenForEmptyRecordList(false). withCleanupLeasesUponShardCompletion(false). withFailoverTimeMillis(20000). withTableName(LEASE_TABLE_PREFIX + dynamoTableName). withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); Worker worker = new Worker.Builder(). recordProcessorFactory(recordProcessorFactory). config(workerConfig). kinesisClient(adapterClient). cloudWatchClient(cloudWatchClient). dynamoDBClient(dynamoDBClient). execService(executor). build(); executor.execute(worker); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }