private void prepare(String processingLanguage) { // Ensure the JVM will refresh the cached IP values of AWS resources (e.g. service endpoints). java.security.Security.setProperty("networkaddress.cache.ttl", "60"); LOG.info("Using workerId: " + kinesisClientLibConfig.getWorkerIdentifier()); LOG.info("Using credentials with access key id: " + kinesisClientLibConfig.getKinesisCredentialsProvider() .getCredentials().getAWSAccessKeyId()); StringBuilder userAgent = new StringBuilder(KinesisClientLibConfiguration.KINESIS_CLIENT_LIB_USER_AGENT); userAgent.append(" "); userAgent.append(USER_AGENT); userAgent.append("/"); userAgent.append(VERSION); if (processingLanguage != null) { userAgent.append(" "); userAgent.append(processingLanguage); } if (recordProcessorFactory.getCommandArray().length > 0) { userAgent.append(" "); userAgent.append(recordProcessorFactory.getCommandArray()[0]); } LOG.info(String.format("MultiLangDaemon is adding the following fields to the User Agent: %s", userAgent.toString())); kinesisClientLibConfig.withUserAgent(userAgent.toString()); }
private static DynamoDBStreamsProxy getDynamoDBStreamsProxy(KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient) { return new DynamoDBStreamsProxy.Builder( config.getStreamName(), config.getKinesisCredentialsProvider(), streamsClient) .build(); }
private static DynamoDBStreamsProxy getDynamoDBStreamsProxy(KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient) { return new DynamoDBStreamsProxy.Builder( config.getStreamName(), config.getKinesisCredentialsProvider(), streamsClient) .build(); }
/** * Public constructor. * @param config */ public KinesisProxy(final KinesisClientLibConfiguration config, final AmazonKinesis client) { this(config.getStreamName(), client, DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES, config.getListShardsBackoffTimeInMillis(), config.getMaxListShardsRetryAttempts()); this.credentialsProvider = config.getKinesisCredentialsProvider(); }
/** * @deprecated The access to this constructor will be changed in a future release. The recommended way to create * a Worker is to use {@link Builder} * * @param recordProcessorFactory * Used to get record processor instances for processing data from shards * @param config * Kinesis Client Library configuration * @param metricsFactory * Metrics factory used to emit metrics * @param execService * ExecutorService to use for processing records (support for multi-threaded consumption) */ @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, IMetricsFactory metricsFactory, ExecutorService execService) { this(recordProcessorFactory, config, new AmazonKinesisClient(config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()), new AmazonDynamoDBClient(config.getDynamoDBCredentialsProvider(), config.getDynamoDBClientConfiguration()), metricsFactory, execService); }
kinesisConfig.getKinesisCredentialsProvider());
/** * Constructor. * * @deprecated The access to this constructor will be changed in a future release. The recommended way to create * a Worker is to use {@link Builder} * * @param recordProcessorFactory * Used to get record processor instances for processing data from shards * @param config * Kinesis Client Library configuration * @param execService * ExecutorService to use for processing records (support for multi-threaded consumption) */ @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, ExecutorService execService) { this(recordProcessorFactory, config, new AmazonKinesisClient(config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()), new AmazonDynamoDBClient(config.getDynamoDBCredentialsProvider(), config.getDynamoDBClientConfiguration()), new AmazonCloudWatchClient(config.getCloudWatchCredentialsProvider(), config.getCloudWatchClientConfiguration()), execService); }
public void start() { int mb = 1024 * 1024; LOG.info("Max memory: {} mb", Runtime.getRuntime().maxMemory() / mb); LOG.info("Starting up Kinesis Consumer... (may take a few seconds)"); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisCfg.getKinesisCredentialsProvider(), kinesisCfg.getKinesisClientConfiguration()); AmazonDynamoDBClient dynamoDBClient = new AmazonDynamoDBClient(kinesisCfg.getDynamoDBCredentialsProvider(), kinesisCfg.getDynamoDBClientConfiguration()); AmazonCloudWatch cloudWatchClient = new AmazonCloudWatchClient(kinesisCfg.getCloudWatchCredentialsProvider(), kinesisCfg.getCloudWatchClientConfiguration()); Worker worker = new Worker.Builder() .recordProcessorFactory(() -> new RecordProcessor(unitOfWorkListener, exceptionStrategy, metricsCallback, dry)) .config(kinesisCfg) .kinesisClient(kinesisClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .build(); worker.run(); }
this.config.getKinesisCredentialsProvider()); if (this.config.getRegionName() != null) { region = Region.getRegion(Regions.fromName(this.config this.config.getKinesisCredentialsProvider()); if (region != null) this.kinesisClient.setRegion(region); .withCredentials(this.config.getKinesisCredentialsProvider()) .withAggregateType(this.aggregatorType) .withTableName(this.tableName).withLabelColumn(labelColumn)
/** * Factory method. * * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, ExecutorService execService) { AmazonDynamoDBStreamsAdapterClient streamsClient = new AmazonDynamoDBStreamsAdapterClient( config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()); return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * Factory method. * * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, ExecutorService execService) { AmazonDynamoDBStreamsAdapterClient streamsClient = new AmazonDynamoDBStreamsAdapterClient( config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()); return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration(), config.getKinesisEndpoint(),