@Override public KinesisCollector start() { String workerId; try { workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); } catch (UnknownHostException e) { workerId = UUID.randomUUID().toString(); } KinesisClientLibConfiguration config = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId); config.withRegionName(regionName); processor = new KinesisRecordProcessorFactory(collector); worker = new Worker.Builder().recordProcessorFactory(processor).config(config).build(); executor.execute(worker); return this; }
@Override public void run() { try { String workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(APPLICATION_NAME, kdsStreamName, credentialsProvider, workerId); kinesisClientLibConfiguration.withInitialPositionInStream(SAMPLE_APPLICATION_INITIAL_POSITION_IN_STREAM).withRegionName(region.getName()); final IRecordProcessorFactory recordProcessorFactory = () -> new KinesisRecordProcessor(rekognizedFragmentsIndex, credentialsProvider); final Worker worker = new Worker(recordProcessorFactory, kinesisClientLibConfiguration); System.out.printf("Running %s to process stream %s as worker %s...", APPLICATION_NAME, kdsStreamName, workerId); int exitCode = 0; try { worker.run(); } catch (Throwable t) { System.err.println("Caught throwable while processing data."); t.printStackTrace(); exitCode = 1; } System.out.println("Exit code : " + exitCode); } catch (Exception e) { e.printStackTrace(); } }
new KinesisClientLibConfiguration(applicationName, streamName, provider, providerDynamoDB, providerCloudWatch, workerId);
workerConfig = new KinesisClientLibConfiguration("streams-adapter-demo", streamArn, streamsCredentials, "streams-demo-worker") .withMaxRecords(1)
kinesisConfig = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId); if (regionName != null) {
@Override protected void doStart() throws FlumeException { try { workerId = InetAddress.getLocalHost().getCanonicalHostName() + ": " + getName(); appName = "Flume Kinesis Source: " + workerId; credentialsProvider = new InstanceProfileCredentialsProvider(); // Needed to verify that we can access credentials. credentialsProvider.getCredentials(); } catch (UnknownHostException e) { throw new FlumeException("Error while finding host", e); } catch (AmazonClientException e) { credentialsProvider = new ProfileCredentialsProvider(); credentialsProvider.getCredentials(); } clientConfig = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId) .withInitialPositionInStream( InitialPositionInStream.TRIM_HORIZON) .withMaxRecords(batchSize); // Maximum batch size. executor.execute(new Runnable() { @Override public void run() { while (!Thread.currentThread().isInterrupted()){ worker = new Worker(processorFactory, clientConfig); worker.run(); // Returns when worker is shutdown } } }); }
/** * Get KCL config for a given system stream. * @param system name of the system * @param stream name of the stream * @param appName name of the application * @return Stream scoped KCL configs required to build * {@link KinesisClientLibConfiguration} */ public KinesisClientLibConfiguration getKinesisClientLibConfig(String system, String stream, String appName) { ClientConfiguration clientConfig = getAWSClientConfig(system); String workerId = appName + "-" + UUID.randomUUID(); InitialPositionInStream startPos = InitialPositionInStream.LATEST; AWSCredentialsProvider provider = credentialsProviderForStream(system, stream); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(appName, stream, provider, workerId) .withRegionName(getRegion(system, stream).getName()) .withKinesisClientConfig(clientConfig) .withCloudWatchClientConfig(clientConfig) .withDynamoDBClientConfig(clientConfig) .withInitialPositionInStream(startPos) .withCallProcessRecordsEvenForEmptyRecordList(true); // For health monitoring metrics. // First, get system scoped configs for KCL and override with configs set at stream scope. setKinesisClientLibConfigs( subset(String.format(CONFIG_SYSTEM_KINESIS_CLIENT_LIB_CONFIG, system)), kinesisClientLibConfiguration); setKinesisClientLibConfigs(subset(String.format(CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, system, stream)), kinesisClientLibConfiguration); return kinesisClientLibConfiguration; }
+ credentialsProvider.getCredentials().getAWSAccessKeyId()); config = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId).withInitialPositionInStream( streamPosition).withKinesisEndpoint(kinesisEndpoint);
initialize(KinesisConnectorConfiguration kinesisConnectorConfiguration, IMetricsFactory metricFactory) { KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(kinesisConnectorConfiguration.APP_NAME, kinesisConnectorConfiguration.KINESIS_INPUT_STREAM, kinesisConnectorConfiguration.AWS_CREDENTIALS_PROVIDER,
initialize(KinesisConnectorConfiguration kinesisConnectorConfiguration, IMetricsFactory metricFactory) { KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(kinesisConnectorConfiguration.APP_NAME, kinesisConnectorConfiguration.KINESIS_INPUT_STREAM, kinesisConnectorConfiguration.AWS_CREDENTIALS_PROVIDER,
final KinesisClientLibConfiguration kclConfig = new KinesisClientLibConfiguration(actualTaskName, streamArn, credentialsProvider,
public void startReplicatingChanges() throws StreamNotEnabledException { if (tableSchema == null) { throw new TableExistsException("table %s does not exist in destination", dynamoTableName); } String tableStreamArn = getStreamArn(); if (tableStreamArn == null) { throw new StreamNotEnabledException("table %s does not have a stream enabled\n", dynamoTableName); } AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient); AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.standard().build(); String workerId = generateWorkerId(); final KinesisClientLibConfiguration workerConfig = new KinesisClientLibConfiguration( APPLICATION_NAME, tableStreamArn, awsCredentialsProvider, workerId). withMaxRecords(1000). withIdleTimeBetweenReadsInMillis(500). withCallProcessRecordsEvenForEmptyRecordList(false). withCleanupLeasesUponShardCompletion(false). withFailoverTimeMillis(20000). withTableName(LEASE_TABLE_PREFIX + dynamoTableName). withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); Worker worker = new Worker.Builder(). recordProcessorFactory(recordProcessorFactory). config(workerConfig). kinesisClient(adapterClient). cloudWatchClient(cloudWatchClient). dynamoDBClient(dynamoDBClient). execService(executor). build(); executor.execute(worker); }