.withMaxRecords(1) .withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON);
@Override protected void doStart() throws FlumeException { try { workerId = InetAddress.getLocalHost().getCanonicalHostName() + ": " + getName(); appName = "Flume Kinesis Source: " + workerId; credentialsProvider = new InstanceProfileCredentialsProvider(); // Needed to verify that we can access credentials. credentialsProvider.getCredentials(); } catch (UnknownHostException e) { throw new FlumeException("Error while finding host", e); } catch (AmazonClientException e) { credentialsProvider = new ProfileCredentialsProvider(); credentialsProvider.getCredentials(); } clientConfig = new KinesisClientLibConfiguration(appName, streamName, credentialsProvider, workerId) .withInitialPositionInStream( InitialPositionInStream.TRIM_HORIZON) .withMaxRecords(batchSize); // Maximum batch size. executor.execute(new Runnable() { @Override public void run() { while (!Thread.currentThread().isInterrupted()){ worker = new Worker(processorFactory, clientConfig); worker.run(); // Returns when worker is shutdown } } }); }
config.withMaxRecords(maxRecords);
kinesisConnectorConfiguration.WORKER_ID).withKinesisEndpoint(kinesisConnectorConfiguration.KINESIS_ENDPOINT) .withFailoverTimeMillis(kinesisConnectorConfiguration.FAILOVER_TIME) .withMaxRecords(kinesisConnectorConfiguration.MAX_RECORDS) .withInitialPositionInStream(kinesisConnectorConfiguration.INITIAL_POSITION_IN_STREAM) .withIdleTimeBetweenReadsInMillis(kinesisConnectorConfiguration.IDLE_TIME_BETWEEN_READS)
kinesisConnectorConfiguration.WORKER_ID).withKinesisEndpoint(kinesisConnectorConfiguration.KINESIS_ENDPOINT) .withFailoverTimeMillis(kinesisConnectorConfiguration.FAILOVER_TIME) .withMaxRecords(kinesisConnectorConfiguration.MAX_RECORDS) .withInitialPositionInStream(kinesisConnectorConfiguration.INITIAL_POSITION_IN_STREAM) .withIdleTimeBetweenReadsInMillis(kinesisConnectorConfiguration.IDLE_TIME_BETWEEN_READS)
.withMaxRecords(getRecordsLimit.or(DynamoDBConnectorConstants.STREAMS_RECORDS_LIMIT))
public void startReplicatingChanges() throws StreamNotEnabledException { if (tableSchema == null) { throw new TableExistsException("table %s does not exist in destination", dynamoTableName); } String tableStreamArn = getStreamArn(); if (tableStreamArn == null) { throw new StreamNotEnabledException("table %s does not have a stream enabled\n", dynamoTableName); } AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient); AmazonCloudWatch cloudWatchClient = AmazonCloudWatchClientBuilder.standard().build(); String workerId = generateWorkerId(); final KinesisClientLibConfiguration workerConfig = new KinesisClientLibConfiguration( APPLICATION_NAME, tableStreamArn, awsCredentialsProvider, workerId). withMaxRecords(1000). withIdleTimeBetweenReadsInMillis(500). withCallProcessRecordsEvenForEmptyRecordList(false). withCleanupLeasesUponShardCompletion(false). withFailoverTimeMillis(20000). withTableName(LEASE_TABLE_PREFIX + dynamoTableName). withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); Worker worker = new Worker.Builder(). recordProcessorFactory(recordProcessorFactory). config(workerConfig). kinesisClient(adapterClient). cloudWatchClient(cloudWatchClient). dynamoDBClient(dynamoDBClient). execService(executor). build(); executor.execute(worker); }