initialize(KinesisConnectorConfiguration kinesisConnectorConfiguration, IMetricsFactory metricFactory) { KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(kinesisConnectorConfiguration.APP_NAME, kinesisConnectorConfiguration.KINESIS_INPUT_STREAM, kinesisConnectorConfiguration.AWS_CREDENTIALS_PROVIDER, kinesisConnectorConfiguration.WORKER_ID).withKinesisEndpoint(kinesisConnectorConfiguration.KINESIS_ENDPOINT) .withFailoverTimeMillis(kinesisConnectorConfiguration.FAILOVER_TIME) .withMaxRecords(kinesisConnectorConfiguration.MAX_RECORDS) .withInitialPositionInStream(kinesisConnectorConfiguration.INITIAL_POSITION_IN_STREAM) .withIdleTimeBetweenReadsInMillis(kinesisConnectorConfiguration.IDLE_TIME_BETWEEN_READS) .withCallProcessRecordsEvenForEmptyRecordList(KinesisConnectorConfiguration.DEFAULT_CALL_PROCESS_RECORDS_EVEN_FOR_EMPTY_LIST) .withCleanupLeasesUponShardCompletion(kinesisConnectorConfiguration.CLEANUP_TERMINATED_SHARDS_BEFORE_EXPIRY) .withParentShardPollIntervalMillis(kinesisConnectorConfiguration.PARENT_SHARD_POLL_INTERVAL) .withShardSyncIntervalMillis(kinesisConnectorConfiguration.SHARD_SYNC_INTERVAL) .withTaskBackoffTimeMillis(kinesisConnectorConfiguration.BACKOFF_INTERVAL) .withMetricsBufferTimeMillis(kinesisConnectorConfiguration.CLOUDWATCH_BUFFER_TIME) .withMetricsMaxQueueSize(kinesisConnectorConfiguration.CLOUDWATCH_MAX_QUEUE_SIZE) .withUserAgent(kinesisConnectorConfiguration.APP_NAME + "," + kinesisConnectorConfiguration.CONNECTOR_DESTINATION + "," + KinesisConnectorConfiguration.KINESIS_CONNECTOR_USER_AGENT) .withRegionName(kinesisConnectorConfiguration.REGION_NAME);
/** * Constructor. * * @deprecated The access to this constructor will be changed in a future release. The recommended way to create * a Worker is to use {@link Builder} * * @param recordProcessorFactory * Used to get record processor instances for processing data from shards * @param config * Kinesis Client Library configuration * @param execService * ExecutorService to use for processing records (support for multi-threaded consumption) */ @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, ExecutorService execService) { this(recordProcessorFactory, config, new AmazonKinesisClient(config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()), new AmazonDynamoDBClient(config.getDynamoDBCredentialsProvider(), config.getDynamoDBClientConfiguration()), new AmazonCloudWatchClient(config.getCloudWatchCredentialsProvider(), config.getCloudWatchClientConfiguration()), execService); }
/** * Public constructor. * @param config */ public KinesisProxy(final KinesisClientLibConfiguration config, final AmazonKinesis client) { this(config.getStreamName(), client, DEFAULT_DESCRIBE_STREAM_BACKOFF_MILLIS, DEFAULT_DESCRIBE_STREAM_RETRY_TIMES, config.getListShardsBackoffTimeInMillis(), config.getMaxListShardsRetryAttempts()); this.credentialsProvider = config.getKinesisCredentialsProvider(); }
private static DynamoDBStreamsProxy getDynamoDBStreamsProxy(KinesisClientLibConfiguration config, AmazonDynamoDBStreamsAdapterClient streamsClient) { return new DynamoDBStreamsProxy.Builder( config.getStreamName(), config.getKinesisCredentialsProvider(), streamsClient) .build(); }
/** * @deprecated The access to this constructor will be changed in a future release. The recommended way to create * a Worker is to use {@link Builder} * * @param recordProcessorFactory * Used to get record processor instances for processing data from shards * @param config * Kinesis Client Library configuration * @param metricsFactory * Metrics factory used to emit metrics * @param execService * ExecutorService to use for processing records (support for multi-threaded consumption) */ @Deprecated public Worker( com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, IMetricsFactory metricsFactory, ExecutorService execService) { this(recordProcessorFactory, config, new AmazonKinesisClient(config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()), new AmazonDynamoDBClient(config.getDynamoDBCredentialsProvider(), config.getDynamoDBClientConfiguration()), metricsFactory, execService); }
config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration(), config.getKinesisEndpoint(), config.getRegionName()); config.getDynamoDBCredentialsProvider(), config.getDynamoDBClientConfiguration(), config.getDynamoDBEndpoint(), config.getRegionName()); config.getCloudWatchCredentialsProvider(), config.getCloudWatchClientConfiguration(), null, config.getRegionName()); if (config.getRegionName() != null) { setField(cloudWatchClient, "region", cloudWatchClient::setRegion, RegionUtils.getRegion(config.getRegionName())); setField(kinesisClient, "region", kinesisClient::setRegion, RegionUtils.getRegion(config.getRegionName())); setField(dynamoDBClient, "region", dynamoDBClient::setRegion, RegionUtils.getRegion(config.getRegionName())); if (config.getDynamoDBEndpoint() != null) { setField(dynamoDBClient, "endpoint", dynamoDBClient::setEndpoint, config.getDynamoDBEndpoint()); if (config.getKinesisEndpoint() != null) { setField(kinesisClient, "endpoint", kinesisClient::setEndpoint, config.getKinesisEndpoint()); leaseManager = new KinesisClientLeaseManager(config.getTableName(), dynamoDBClient); return new Worker(config.getApplicationName(),
+ credentialsProvider.getCredentials().getAWSAccessKeyId()); config = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId).withInitialPositionInStream( streamPosition).withKinesisEndpoint(kinesisEndpoint); config.getKinesisClientConfiguration().setUserAgent( StreamAggregator.AWSApplication); config.withRegionName(region.getName()); config.withMaxRecords(maxRecords); config.getApplicationName(), config.getStreamName(), config.getRegionName(), config.getWorkerIdentifier(), config.getMaxRecords()));
KinesisClientLibConfiguration config, AmazonKinesis kinesisClient, AmazonDynamoDB dynamoDBClient, IMetricsFactory metricsFactory, ExecutorService execService) { this(config.getApplicationName(), new V1ToV2RecordProcessorFactoryAdapter(recordProcessorFactory), config, new StreamConfig( new KinesisProxy(config, kinesisClient), config.getMaxRecords(), config.getIdleTimeBetweenReadsInMillis(), config.shouldCallProcessRecordsEvenForEmptyRecordList(), config.shouldValidateSequenceNumberBeforeCheckpointing(), config.getInitialPositionInStreamExtended()), config.getInitialPositionInStreamExtended(), config.getParentShardPollIntervalMillis(), config.getShardSyncIntervalMillis(), config.shouldCleanupLeasesUponShardCompletion(), null, new KinesisClientLibLeaseCoordinator( new KinesisClientLeaseManager(config.getTableName(), dynamoDBClient), config.getWorkerIdentifier(), config.getFailoverTimeMillis(), config.getEpsilonMillis(), config.getMaxLeasesForWorker(), config.getMaxLeasesToStealAtOneTime(), config.getMaxLeaseRenewalThreads(), metricsFactory) .withInitialLeaseTableReadCapacity(config.getInitialLeaseTableReadCapacity()) .withInitialLeaseTableWriteCapacity(config.getInitialLeaseTableWriteCapacity()), execService, metricsFactory, config.getTaskBackoffTimeMillis(), config.getFailoverTimeMillis(), config.getSkipShardSyncAtWorkerInitializationIfLeasesExist(), config.getShardPrioritizationStrategy(),
public void startReplicatingChanges() throws StreamNotEnabledException { if (tableSchema == null) { throw new TableExistsException("table %s does not exist in destination", dynamoTableName); } String tableStreamArn = getStreamArn(); if (tableStreamArn == null) { throw new StreamNotEnabledException("table %s does not have a stream enabled\n", dynamoTableName); } AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient); AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.standard().build(); String workerId = generateWorkerId(); final KinesisClientLibConfiguration workerConfig = new KinesisClientLibConfiguration( APPLICATION_NAME, tableStreamArn, awsCredentialsProvider, workerId). withMaxRecords(1000). withIdleTimeBetweenReadsInMillis(500). withCallProcessRecordsEvenForEmptyRecordList(false). withCleanupLeasesUponShardCompletion(false). withFailoverTimeMillis(20000). withTableName(LEASE_TABLE_PREFIX + dynamoTableName). withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); Worker worker = new Worker.Builder(). recordProcessorFactory(recordProcessorFactory). config(workerConfig). kinesisClient(adapterClient). cloudWatchClient(cloudWatchClient). dynamoDBClient(dynamoDBClient). execService(executor). build(); executor.execute(worker); }
kinesisConfig = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId); if (regionName != null) { kinesisConfig.withRegionName(regionName); kinesisConfig.getKinesisCredentialsProvider()); if (kinesisConfig.getRegionName() != null) { asyncClient.setRegion(Region.getRegion(Regions.fromName(kinesisConfig.getRegionName()))); logger.info("appName : {}", kinesisConfig.getApplicationName()); logger.info("streamName: {}", kinesisConfig.getStreamName()); logger.info("regionName: {}", kinesisConfig.getRegionName()); logger.info("workerId : {}", kinesisConfig.getWorkerIdentifier()); logger.info("streamArn : {}", bridge.getStreamArn());
/** * Get KCL config for a given system stream. * @param system name of the system * @param stream name of the stream * @param appName name of the application * @return Stream scoped KCL configs required to build * {@link KinesisClientLibConfiguration} */ public KinesisClientLibConfiguration getKinesisClientLibConfig(String system, String stream, String appName) { ClientConfiguration clientConfig = getAWSClientConfig(system); String workerId = appName + "-" + UUID.randomUUID(); InitialPositionInStream startPos = InitialPositionInStream.LATEST; AWSCredentialsProvider provider = credentialsProviderForStream(system, stream); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(appName, stream, provider, workerId) .withRegionName(getRegion(system, stream).getName()) .withKinesisClientConfig(clientConfig) .withCloudWatchClientConfig(clientConfig) .withDynamoDBClientConfig(clientConfig) .withInitialPositionInStream(startPos) .withCallProcessRecordsEvenForEmptyRecordList(true); // For health monitoring metrics. // First, get system scoped configs for KCL and override with configs set at stream scope. setKinesisClientLibConfigs( subset(String.format(CONFIG_SYSTEM_KINESIS_CLIENT_LIB_CONFIG, system)), kinesisClientLibConfiguration); setKinesisClientLibConfigs(subset(String.format(CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, system, stream)), kinesisClientLibConfiguration); return kinesisClientLibConfiguration; }
final KinesisClientLibConfiguration kclConfig = new KinesisClientLibConfiguration(actualTaskName, streamArn, credentialsProvider, .withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON) .withMaxRecords(getRecordsLimit.or(DynamoDBConnectorConstants.STREAMS_RECORDS_LIMIT)) .withIdleTimeBetweenReadsInMillis(DynamoDBConnectorConstants.IDLE_TIME_BETWEEN_READS) .withValidateSequenceNumberBeforeCheckpointing(false) .withParentShardPollIntervalMillis(parentShardPollIntervalMillis.or(DynamoDBConnectorConstants.DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS)) .withFailoverTimeMillis(DynamoDBConnectorConstants.KCL_FAILOVER_TIME);
@Override protected void doStart() throws FlumeException { try { workerId = InetAddress.getLocalHost().getCanonicalHostName() + ": " + getName(); appName = "Flume Kinesis Source: " + workerId; credentialsProvider = new InstanceProfileCredentialsProvider(); // Needed to verify that we can access credentials. credentialsProvider.getCredentials(); } catch (UnknownHostException e) { throw new FlumeException("Error while finding host", e); } catch (AmazonClientException e) { credentialsProvider = new ProfileCredentialsProvider(); credentialsProvider.getCredentials(); } clientConfig = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId) .withInitialPositionInStream( InitialPositionInStream.TRIM_HORIZON) .withMaxRecords(batchSize); // Maximum batch size. executor.execute(new Runnable() { @Override public void run() { while (!Thread.currentThread().isInterrupted()){ worker = new Worker(processorFactory, clientConfig); worker.run(); // Returns when worker is shutdown } } }); }
this.config.getKinesisCredentialsProvider()); if (this.config.getRegionName() != null) { region = Region.getRegion(Regions.fromName(this.config .getRegionName())); kinesisClient.setRegion(region); kinesisClient, this.config.getStreamName()))) { this.isFirstShardWorker = true; logInfo("Aggregator taking Primary Thread Responsibility"); .withSocketTimeout(60000); this.dynamoClient = new AmazonDynamoDBAsyncClient( this.config.getDynamoDBCredentialsProvider(), clientConfig); if (region != null) this.dynamoClient.setRegion(region); this.config.getKinesisCredentialsProvider()); if (region != null) this.kinesisClient.setRegion(region); config.getApplicationName(), this.getNamespace()); .withCredentials(this.config.getKinesisCredentialsProvider()) .withAggregateType(this.aggregatorType) .withTableName(this.tableName).withLabelColumn(labelColumn) this.config.getCloudWatchCredentialsProvider()); if (this.config.getRegionName() != null)
@Override public void run() { try { String workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(APPLICATION_NAME, kdsStreamName, credentialsProvider, workerId); kinesisClientLibConfiguration.withInitialPositionInStream(SAMPLE_APPLICATION_INITIAL_POSITION_IN_STREAM).withRegionName(region.getName()); final IRecordProcessorFactory recordProcessorFactory = () -> new KinesisRecordProcessor(rekognizedFragmentsIndex, credentialsProvider); final Worker worker = new Worker(recordProcessorFactory, kinesisClientLibConfiguration); System.out.printf("Running %s to process stream %s as worker %s...", APPLICATION_NAME, kdsStreamName, workerId); int exitCode = 0; try { worker.run(); } catch (Throwable t) { System.err.println("Caught throwable while processing data."); t.printStackTrace(); exitCode = 1; } System.out.println("Exit code : " + exitCode); } catch (Exception e) { e.printStackTrace(); } }
@Override public KinesisCollector start() { String workerId; try { workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); } catch (UnknownHostException e) { workerId = UUID.randomUUID().toString(); } KinesisClientLibConfiguration config = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId); config.withRegionName(regionName); processor = new KinesisRecordProcessorFactory(collector); worker = new Worker.Builder().recordProcessorFactory(processor).config(config).build(); executor.execute(worker); return this; }
/** * Factory method. * * @param recordProcessorFactory Used to get record processor instances for processing data from shards * @param config Kinesis Client Library configuration * @param execService ExecutorService to use for processing records (support for multi-threaded * consumption) */ public static Worker createDynamoDbStreamsWorker(IRecordProcessorFactory recordProcessorFactory, KinesisClientLibConfiguration config, ExecutorService execService) { AmazonDynamoDBStreamsAdapterClient streamsClient = new AmazonDynamoDBStreamsAdapterClient( config.getKinesisCredentialsProvider(), config.getKinesisClientConfiguration()); return new Worker .Builder() .recordProcessorFactory(recordProcessorFactory) .config(config) .kinesisClient(streamsClient) .execService(execService) .kinesisProxy(getDynamoDBStreamsProxy(config, streamsClient)) .shardPrioritization(config.getShardPrioritizationStrategy()) .build(); }
/** * @param propertiesFile The location of the properties file. * @param classLoader A classloader, useful if trying to programmatically configure with the daemon, such as in a * unit test. * @param configurator A configurator to use. * @throws IOException Thrown when the properties file can't be accessed. * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected. */ public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader, KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException { Properties properties = loadProperties(classLoader, propertiesFile); if (!validateProperties(properties)) { throw new IllegalArgumentException( "Must provide an executable name in the properties file, " + "e.g. executableName = sampleapp.py"); } String executableName = properties.getProperty(PROP_EXECUTABLE_NAME); String processingLanguage = properties.getProperty(PROP_PROCESSING_LANGUAGE); ClientConfiguration clientConfig = buildClientConfig(properties); kinesisClientLibConfig = configurator.getConfiguration(properties).withKinesisClientConfig(clientConfig) .withCloudWatchClientConfig(clientConfig).withDynamoDBClientConfig(clientConfig); executorService = buildExecutorService(properties); recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService, kinesisClientLibConfig); LOG.info("Running " + kinesisClientLibConfig.getApplicationName() + " to process stream " + kinesisClientLibConfig.getStreamName() + " with executable " + executableName); prepare(processingLanguage); }
/** * Given configuration, returns appropriate metrics factory. * * @param cloudWatchClient * Amazon CloudWatch client * @param config * KinesisClientLibConfiguration * @return Returns metrics factory based on the config. */ private static IMetricsFactory getMetricsFactory(AmazonCloudWatch cloudWatchClient, KinesisClientLibConfiguration config) { IMetricsFactory metricsFactory; if (config.getMetricsLevel() == MetricsLevel.NONE) { metricsFactory = new NullMetricsFactory(); } else { if (config.getRegionName() != null) { setField(cloudWatchClient, "region", cloudWatchClient::setRegion, RegionUtils.getRegion(config.getRegionName())); } metricsFactory = new WorkerCWMetricsFactory(cloudWatchClient, config.getApplicationName(), config.getMetricsBufferTimeMillis(), config.getMetricsMaxQueueSize(), config.getMetricsLevel(), config.getMetricsEnabledDimensions()); } return metricsFactory; }
new KinesisClientLibConfiguration(applicationName, streamName, provider, providerDynamoDB, providerCloudWatch, workerId);