public KinesisReactorBridge start() { logger.info("starting {}...", this); IRecordProcessorFactory factory = new IRecordProcessorFactory() { @Override public IRecordProcessor createProcessor() { BridgeRecordProcessor p = new BridgeRecordProcessor(); logger.info("creating {}", p); return p; } }; Preconditions.checkNotNull(kinesisConfig); worker = new Worker.Builder() .recordProcessorFactory(factory) .config(kinesisConfig) .build(); Thread t = new Thread(worker); t.setDaemon(true); t.setName("kinesis-bridge-" + bridgeThreadNum.getAndIncrement()); t.start(); return this; }
public void start() { int mb = 1024 * 1024; LOG.info("Max memory: {} mb", Runtime.getRuntime().maxMemory() / mb); LOG.info("Starting up Kinesis Consumer... (may take a few seconds)"); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisCfg.getKinesisCredentialsProvider(), kinesisCfg.getKinesisClientConfiguration()); AmazonDynamoDBClient dynamoDBClient = new AmazonDynamoDBClient(kinesisCfg.getDynamoDBCredentialsProvider(), kinesisCfg.getDynamoDBClientConfiguration()); AmazonCloudWatch cloudWatchClient = new AmazonCloudWatchClient(kinesisCfg.getCloudWatchCredentialsProvider(), kinesisCfg.getCloudWatchClientConfiguration()); Worker worker = new Worker.Builder() .recordProcessorFactory(() -> new RecordProcessor(unitOfWorkListener, exceptionStrategy, metricsCallback, dry)) .config(kinesisCfg) .kinesisClient(kinesisClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .build(); worker.run(); }
public void startReplicatingChanges() throws StreamNotEnabledException { if (tableSchema == null) { throw new TableExistsException("table %s does not exist in destination", dynamoTableName); } String tableStreamArn = getStreamArn(); if (tableStreamArn == null) { throw new StreamNotEnabledException("table %s does not have a stream enabled\n", dynamoTableName); } AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient); AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.standard().build(); String workerId = generateWorkerId(); final KinesisClientLibConfiguration workerConfig = new KinesisClientLibConfiguration( APPLICATION_NAME, tableStreamArn, awsCredentialsProvider, workerId). withMaxRecords(1000). withIdleTimeBetweenReadsInMillis(500). withCallProcessRecordsEvenForEmptyRecordList(false). withCleanupLeasesUponShardCompletion(false). withFailoverTimeMillis(20000). withTableName(LEASE_TABLE_PREFIX + dynamoTableName). withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); Worker worker = new Worker.Builder(). recordProcessorFactory(recordProcessorFactory). config(workerConfig). kinesisClient(adapterClient). cloudWatchClient(cloudWatchClient). dynamoDBClient(dynamoDBClient). execService(executor). build(); executor.execute(worker); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param metricsFactory Metrics factory used to emit metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, IMetricsFactory metricsFactory, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .metricsFactory(metricsFactory) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * Factory method. * * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, ExecutorService execService) { AmazonDynamoDBStreamsAdapterClient streamsClient = new AmazonDynamoDBStreamsAdapterClient( config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()); return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param metricsFactory Metrics factory used to emit metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, IMetricsFactory metricsFactory, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .metricsFactory(metricsFactory) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
@Override public void start() { LOG.info("Start samza consumer for system {}.", system); metrics.initializeMetrics(streams); ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() .setNameFormat("kinesis-worker-thread-" + system + "-%d") .build(); // launch kinesis workers in separate threads, one per stream executorService = Executors.newFixedThreadPool(streams.size(), namedThreadFactory); for (String stream : streams) { // KCL Dynamodb table is used for storing the state of processing. By default, the table name is the same as the // application name. Dynamodb table name must be unique for a given account and region (even across different // streams). So, let's create the default one with the combination of job name, job id and stream name. The table // name could be changed by providing a different TableName via KCL specific config. String kinesisApplicationName = kConfig.get(JobConfig.JOB_NAME()) + "-" + kConfig.get(JobConfig.JOB_ID()) + "-" + stream; Worker worker = new Worker.Builder() .recordProcessorFactory(createRecordProcessorFactory(stream)) .config(kConfig.getKinesisClientLibConfig(system, stream, kinesisApplicationName)) .build(); workers.add(worker); // launch kinesis workers in separate thread-pools, one per stream executorService.execute(worker); LOG.info("Started worker for system {} stream {}.", system, stream); } }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param metricsFactory Metrics factory used to emit metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, IMetricsFactory metricsFactory, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .metricsFactory(metricsFactory) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * Factory method. * * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, ExecutorService execService) { AmazonDynamoDBStreamsAdapterClient streamsClient = new AmazonDynamoDBStreamsAdapterClient( config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()); return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param metricsFactory Metrics factory used to emit metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, IMetricsFactory metricsFactory, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .metricsFactory(metricsFactory) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient, ExecutorService execService) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDB dynamoDBClient, AmazonCloudWatch cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param streamsClient DynamoDB Streams Adapter Client used for fetching data * @param dynamoDBClient DynamoDB client used for checkpoints and tracking leases * @param cloudWatchClient CloudWatch Client for publishing metrics */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient, AmazonDynamoDBClient dynamoDBClient, AmazonCloudWatchClient cloudWatchClient) { return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
@Override public KinesisCollector start() { String workerId; try { workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); } catch (UnknownHostException e) { workerId = UUID.randomUUID().toString(); } KinesisClientLibConfiguration config = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId); config.withRegionName(regionName); processor = new KinesisRecordProcessorFactory(collector); worker = new Worker.Builder().recordProcessorFactory(processor).config(config).build(); executor.execute(worker); return this; }
private static Worker buildWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration configuration, ExecutorService workerThreadPool) { return new Worker.Builder().recordProcessorFactory(recordProcessorFactory).config(configuration) .execService(workerThreadPool).build(); }