Refine search
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getStreamName() == null) ? 0 : getStreamName().hashCode()); hashCode = prime * hashCode + ((getStreamARN() == null) ? 0 : getStreamARN().hashCode()); hashCode = prime * hashCode + ((getStreamStatus() == null) ? 0 : getStreamStatus().hashCode()); hashCode = prime * hashCode + ((getShards() == null) ? 0 : getShards().hashCode()); hashCode = prime * hashCode + ((getHasMoreShards() == null) ? 0 : getHasMoreShards().hashCode()); hashCode = prime * hashCode + ((getRetentionPeriodHours() == null) ? 0 : getRetentionPeriodHours().hashCode()); hashCode = prime * hashCode + ((getStreamCreationTimestamp() == null) ? 0 : getStreamCreationTimestamp().hashCode()); hashCode = prime * hashCode + ((getEnhancedMonitoring() == null) ? 0 : getEnhancedMonitoring().hashCode()); hashCode = prime * hashCode + ((getEncryptionType() == null) ? 0 : getEncryptionType().hashCode()); hashCode = prime * hashCode + ((getKeyId() == null) ? 0 : getKeyId().hashCode()); return hashCode; }
public StreamDescription unmarshall(JsonUnmarshallerContext context) throws Exception { StreamDescription streamDescription = new StreamDescription(); if (context.testExpression("StreamName", targetDepth)) { context.nextToken(); streamDescription.setStreamName(context.getUnmarshaller(String.class).unmarshall(context)); streamDescription.setStreamARN(context.getUnmarshaller(String.class).unmarshall(context)); streamDescription.setStreamStatus(context.getUnmarshaller(String.class).unmarshall(context)); streamDescription.setShards(new ListUnmarshaller<Shard>(ShardJsonUnmarshaller.getInstance()).unmarshall(context)); streamDescription.setHasMoreShards(context.getUnmarshaller(Boolean.class).unmarshall(context)); streamDescription.setRetentionPeriodHours(context.getUnmarshaller(Integer.class).unmarshall(context)); streamDescription.setStreamCreationTimestamp(context.getUnmarshaller(java.util.Date.class).unmarshall(context)); streamDescription.setEnhancedMonitoring(new ListUnmarshaller<EnhancedMetrics>(EnhancedMetricsJsonUnmarshaller.getInstance()) .unmarshall(context)); streamDescription.setEncryptionType(context.getUnmarshaller(String.class).unmarshall(context)); streamDescription.setKeyId(context.getUnmarshaller(String.class).unmarshall(context));
List<Shard> getShardsForStream (String stream) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(stream); List<Shard> shards = new ArrayList<>(); String exclusiveStartShardId = null; do { describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while ( exclusiveStartShardId != null ); LOG.info("Number of shards for stream " + stream + " are " + shards.size()); return shards; }
describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId); describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest); StreamDescription streamDescription = describeStreamResult.getStreamDescription(); if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) { shardList.addAll(streamDescription.getShards()); if (streamDescription.getHasMoreShards()) { exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId(); .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) { ResourceNotFoundException resourceNotFoundException =
@Override public Set<String> getPartitionIds(String stream) { checkIfClosed(); return kinesis.describeStream(stream) .getStreamDescription() .getShards() .stream() .map(Shard::getShardId).collect(Collectors.toSet()); }
private ShardGraphProcessingResult buildShardGraphSnapshot() { DescribeStreamResult response; do { response = getStreamInfo(shardGraph.getLastFetchedShardId()); if (response == null) { /* * If getStreamInfo ever returns null, we should bail and return null from getShardList. * This indicates the stream is not in ACTIVE state and we may not have accurate/consistent information * about the stream. By returning ShardGraphProcessingResult.STREAM_DISABLED from here, we indicate that * getStreamInfo returned a null response and the caller (getShardList) should return null. If, on the * other hand, an exception is thrown from getStreamInfo, it will bubble up to the caller of * getShardList, which then handles it accordingly. */ return ShardGraphProcessingResult.STREAM_DISABLED; } else { shardGraph.addNodes(response.getStreamDescription().getShards()); LOG.debug(String.format("Building shard graph snapshot; total shard count: %d", shardGraph.size())); } } while (response.getStreamDescription().isHasMoreShards()); return ShardGraphProcessingResult.FETCHED_ALL_AVAILABLE_SHARDS; }
/** * Return the state of a Amazon Kinesis stream. * * @param kinesisClient The {@link com.amazonaws.services.kinesis.AmazonKinesisClient} with Amazon Kinesis read privileges * @param streamName The Amazon Kinesis stream to get the state of * @return String representation of the Stream state */ private static String streamState(AmazonKinesisClient kinesisClient, String streamName) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); try { return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus(); } catch (AmazonServiceException e) { return null; } }
@Override public void startStream() { this.streamNames = this.config.getStreams(); for (final String stream : streamNames) { DescribeStreamResult describeStreamResult = client.describeStream(stream); if( "ACTIVE".equals(describeStreamResult.getStreamDescription().getStreamStatus())) { List<Shard> shardList = describeStreamResult.getStreamDescription().getShards(); for( Shard shard : shardList ) { executor.submit(new KinesisPersistReaderTask(this, stream, shard.getShardId())); } } } }
private void project(StreamDescription description) { ObjectNode n = mapper.createObjectNode(); n.put("aws_account", getAccountId()); n.put("aws_region", getRegion().getName()); n.put("aws_arn",description.getStreamARN()); n.put("name",description.getStreamName()); n.put("aws_name", description.getStreamName()); n.put("aws_status", description.getStreamStatus()); n.put("aws_streamCreationTimestamp",description.getStreamCreationTimestamp().getTime()); n.put("aws_retentionPeriodHours",description.getRetentionPeriodHours()); n.put("aws_shardCount", description.getShards().size()); incrementEntityCount(); String cypher = "merge (k:AwsKinesisStream {aws_arn:{aws_arn}}) set k+={props}, k.updateTs=timestamp() return k"; getNeoRxClient().execCypher(cypher, "aws_arn", n.path("aws_arn").asText(), "props", n); cypher = "match (a:AwsAccount {aws_account:{account}}), (k:AwsKinesisStream {aws_account:{account}}) MERGE (a)-[r:OWNS]->(k) set r.updateTs=timestamp()"; getNeoRxClient().execCypher(cypher, "account", getAccountId()); }
@Override public String get() { return getKinesisClient().describeStream(getStreamName()).getStreamDescription().getStreamARN(); }
/** * <p> * The shards that comprise the stream. * </p> * <p> * Returns a reference to this object so that method calls can be chained * together. * * @param shards <p> * The shards that comprise the stream. * </p> * @return A reference to this updated object so that method calls can be * chained together. */ public StreamDescription withShards(Shard... shards) { if (getShards() == null) { this.shards = new java.util.ArrayList<Shard>(shards.length); } for (Shard value : shards) { this.shards.add(value); } return this; }
new DescribeStreamResult() .withStreamDescription(new StreamDescription() .withStreamName(STREAM_FOR_RESHARDING) .withStreamStatus(StreamStatus.ACTIVE) .withHasMoreShards(false) .withShards(new Shard() .withShardId("closedShard") .withSequenceNumberRange(new SequenceNumberRange()
@Override public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) { int nextShardId = 0; if (exclusiveStartShardId != null) { nextShardId = parseInt(exclusiveStartShardId) + 1; } boolean hasMoreShards = nextShardId + 1 < shardedData.size(); List<Shard> shards = new ArrayList<>(); if (nextShardId < shardedData.size()) { shards.add(new Shard().withShardId(Integer.toString(nextShardId))); } HttpResponse response = new HttpResponse(null, null); response.setStatusCode(200); DescribeStreamResult result = new DescribeStreamResult(); result.setSdkHttpMetadata(SdkHttpMetadata.from(response)); result.withStreamDescription( new StreamDescription() .withHasMoreShards(hasMoreShards) .withShards(shards) .withStreamName(streamName)); return result; }
@Test public void shouldListAllShards() throws Exception { Shard shard1 = new Shard().withShardId(SHARD_1); Shard shard2 = new Shard().withShardId(SHARD_2); Shard shard3 = new Shard().withShardId(SHARD_3); given(kinesis.describeStream(STREAM, null)) .willReturn( new DescribeStreamResult() .withStreamDescription( new StreamDescription().withShards(shard1, shard2).withHasMoreShards(true))); given(kinesis.describeStream(STREAM, SHARD_2)) .willReturn( new DescribeStreamResult() .withStreamDescription( new StreamDescription().withShards(shard3).withHasMoreShards(false))); List<Shard> shards = underTest.listShards(STREAM); assertThat(shards).containsOnly(shard1, shard2, shard3); }
describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId); describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest); StreamDescription streamDescription = describeStreamResult.getStreamDescription(); if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) { shardList.addAll(streamDescription.getShards()); if (streamDescription.getHasMoreShards()) { exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId(); .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) { ResourceNotFoundException resourceNotFoundException =
/** * Get the available shards from the kinesis * @param streamName Name of the stream from where the shards to be accessed * @return the list of shards from the given stream */ public List<Shard> getShardList(String streamName) { assert client != null : "Illegal client"; DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.setStreamName(streamName); DescribeStreamResult describeResponse = client.describeStream(describeRequest); return describeResponse.getStreamDescription().getShards(); }
private ShardGraphProcessingResult buildShardGraphSnapshot() { DescribeStreamResult response; do { response = getStreamInfo(shardGraph.getLastFetchedShardId()); if (response == null) { /* * If getStreamInfo ever returns null, we should bail and return null from getShardList. * This indicates the stream is not in ACTIVE state and we may not have accurate/consistent information * about the stream. By returning ShardGraphProcessingResult.STREAM_DISABLED from here, we indicate that * getStreamInfo returned a null response and the caller (getShardList) should return null. If, on the * other hand, an exception is thrown from getStreamInfo, it will bubble up to the caller of * getShardList, which then handles it accordingly. */ return ShardGraphProcessingResult.STREAM_DISABLED; } else { shardGraph.addNodes(response.getStreamDescription().getShards()); LOG.debug(String.format("Building shard graph snapshot; total shard count: %d", shardGraph.size())); } } while (response.getStreamDescription().isHasMoreShards()); return ShardGraphProcessingResult.FETCHED_ALL_AVAILABLE_SHARDS; }
/** * Return the state of a Amazon Kinesis stream. * * @param kinesisClient * The {@link com.amazonaws.services.kinesis.AmazonKinesisClient} with Amazon Kinesis read privileges * @param streamName * The Amazon Kinesis stream to get the state of * @return String representation of the Stream state */ private static String streamState(AmazonKinesisClient kinesisClient, String streamName) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); try { return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus(); } catch (AmazonServiceException e) { return null; } }
public List<Shard> listShards(final String streamName) throws TransientKinesisException { return wrapExceptions( () -> { List<Shard> shards = Lists.newArrayList(); String lastShardId = null; StreamDescription description; do { description = kinesis.describeStream(streamName, lastShardId).getStreamDescription(); shards.addAll(description.getShards()); lastShardId = shards.get(shards.size() - 1).getShardId(); } while (description.getHasMoreShards()); return shards; }); }
.willReturn(new DescribeStreamResult() .withStreamDescription(new StreamDescription() .withStreamName(STREAM1) .withStreamStatus(StreamStatus.UPDATING)), new DescribeStreamResult() .withStreamDescription(new StreamDescription() .withStreamName(STREAM1) .withStreamStatus(StreamStatus.ACTIVE) .withHasMoreShards(false) .withShards(new Shard() .withShardId("1") .withSequenceNumberRange(new SequenceNumberRange()),