@Override public void run() { try { String workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(APPLICATION_NAME, kdsStreamName, credentialsProvider, workerId); kinesisClientLibConfiguration.withInitialPositionInStream(SAMPLE_APPLICATION_INITIAL_POSITION_IN_STREAM).withRegionName(region.getName()); final IRecordProcessorFactory recordProcessorFactory = () -> new KinesisRecordProcessor(rekognizedFragmentsIndex, credentialsProvider); final Worker worker = new Worker(recordProcessorFactory, kinesisClientLibConfiguration); System.out.printf("Running %s to process stream %s as worker %s...", APPLICATION_NAME, kdsStreamName, workerId); int exitCode = 0; try { worker.run(); } catch (Throwable t) { System.err.println("Caught throwable while processing data."); t.printStackTrace(); exitCode = 1; } System.out.println("Exit code : " + exitCode); } catch (Exception e) { e.printStackTrace(); } }
streamArn, streamsCredentials, "streams-demo-worker") .withMaxRecords(1) .withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON);
@Override protected void doStart() throws FlumeException { try { workerId = InetAddress.getLocalHost().getCanonicalHostName() + ": " + getName(); appName = "Flume Kinesis Source: " + workerId; credentialsProvider = new InstanceProfileCredentialsProvider(); // Needed to verify that we can access credentials. credentialsProvider.getCredentials(); } catch (UnknownHostException e) { throw new FlumeException("Error while finding host", e); } catch (AmazonClientException e) { credentialsProvider = new ProfileCredentialsProvider(); credentialsProvider.getCredentials(); } clientConfig = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId) .withInitialPositionInStream( InitialPositionInStream.TRIM_HORIZON) .withMaxRecords(batchSize); // Maximum batch size. executor.execute(new Runnable() { @Override public void run() { while (!Thread.currentThread().isInterrupted()){ worker = new Worker(processorFactory, clientConfig); worker.run(); // Returns when worker is shutdown } } }); }
/** * Get KCL config for a given system stream. * @param system name of the system * @param stream name of the stream * @param appName name of the application * @return Stream scoped KCL configs required to build * {@link KinesisClientLibConfiguration} */ public KinesisClientLibConfiguration getKinesisClientLibConfig(String system, String stream, String appName) { ClientConfiguration clientConfig = getAWSClientConfig(system); String workerId = appName + "-" + UUID.randomUUID(); InitialPositionInStream startPos = InitialPositionInStream.LATEST; AWSCredentialsProvider provider = credentialsProviderForStream(system, stream); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(appName, stream, provider, workerId) .withRegionName(getRegion(system, stream).getName()) .withKinesisClientConfig(clientConfig) .withCloudWatchClientConfig(clientConfig) .withDynamoDBClientConfig(clientConfig) .withInitialPositionInStream(startPos) .withCallProcessRecordsEvenForEmptyRecordList(true); // For health monitoring metrics. // First, get system scoped configs for KCL and override with configs set at stream scope. setKinesisClientLibConfigs( subset(String.format(CONFIG_SYSTEM_KINESIS_CLIENT_LIB_CONFIG, system)), kinesisClientLibConfiguration); setKinesisClientLibConfigs(subset(String.format(CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, system, stream)), kinesisClientLibConfiguration); return kinesisClientLibConfiguration; }
credentialsProvider, workerId).withInitialPositionInStream( streamPosition).withKinesisEndpoint(kinesisEndpoint);
.withFailoverTimeMillis(kinesisConnectorConfiguration.FAILOVER_TIME) .withMaxRecords(kinesisConnectorConfiguration.MAX_RECORDS) .withInitialPositionInStream(kinesisConnectorConfiguration.INITIAL_POSITION_IN_STREAM) .withIdleTimeBetweenReadsInMillis(kinesisConnectorConfiguration.IDLE_TIME_BETWEEN_READS) .withCallProcessRecordsEvenForEmptyRecordList(KinesisConnectorConfiguration.DEFAULT_CALL_PROCESS_RECORDS_EVEN_FOR_EMPTY_LIST)
.withFailoverTimeMillis(kinesisConnectorConfiguration.FAILOVER_TIME) .withMaxRecords(kinesisConnectorConfiguration.MAX_RECORDS) .withInitialPositionInStream(kinesisConnectorConfiguration.INITIAL_POSITION_IN_STREAM) .withIdleTimeBetweenReadsInMillis(kinesisConnectorConfiguration.IDLE_TIME_BETWEEN_READS) .withCallProcessRecordsEvenForEmptyRecordList(KinesisConnectorConfiguration.DEFAULT_CALL_PROCESS_RECORDS_EVEN_FOR_EMPTY_LIST)
.withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON)
public void startReplicatingChanges() throws StreamNotEnabledException { if (tableSchema == null) { throw new TableExistsException("table %s does not exist in destination", dynamoTableName); } String tableStreamArn = getStreamArn(); if (tableStreamArn == null) { throw new StreamNotEnabledException("table %s does not have a stream enabled\n", dynamoTableName); } AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient); AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.standard().build(); String workerId = generateWorkerId(); final KinesisClientLibConfiguration workerConfig = new KinesisClientLibConfiguration( APPLICATION_NAME, tableStreamArn, awsCredentialsProvider, workerId). withMaxRecords(1000). withIdleTimeBetweenReadsInMillis(500). withCallProcessRecordsEvenForEmptyRecordList(false). withCleanupLeasesUponShardCompletion(false). withFailoverTimeMillis(20000). withTableName(LEASE_TABLE_PREFIX + dynamoTableName). withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); Worker worker = new Worker.Builder(). recordProcessorFactory(recordProcessorFactory). config(workerConfig). kinesisClient(adapterClient). cloudWatchClient(cloudWatchClient). dynamoDBClient(dynamoDBClient). execService(executor). build(); executor.execute(worker); }