@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
public Shard unmarshall(JsonUnmarshallerContext context) throws Exception { Shard shard = new Shard(); if (context.testExpression("ShardId", targetDepth)) { context.nextToken(); shard.setShardId(context.getUnmarshaller(String.class).unmarshall(context)); shard.setParentShardId(context.getUnmarshaller(String.class).unmarshall(context)); shard.setAdjacentParentShardId(context.getUnmarshaller(String.class).unmarshall(context)); shard.setHashKeyRange(HashKeyRangeJsonUnmarshaller.getInstance().unmarshall(context)); shard.setSequenceNumberRange(SequenceNumberRangeJsonUnmarshaller.getInstance().unmarshall(context));
void initialize (int myTaskIndex, int totalTasks) { deactivated = false; lastCommitTime = System.currentTimeMillis(); kinesisConnection.initialize(); zkConnection.initialize(); List<Shard> shards = kinesisConnection.getShardsForStream(kinesisConfig.getStreamName()); LOG.info("myTaskIndex is " + myTaskIndex); LOG.info("totalTasks is " + totalTasks); int i = myTaskIndex; while (i < shards.size()) { LOG.info("Shard id " + shards.get(i).getShardId() + " assigned to task " + myTaskIndex); toEmitPerShard.put(shards.get(i).getShardId(), new LinkedList<Record>()); i += totalTasks; } initializeFetchedSequenceNumbers(); refreshShardIteratorsForNewRecords(); }
private Set<Shard> findNextShards(List<Shard> allShards, Set<Shard> expiredShards) { Set<Shard> nextShards = new HashSet<>(); for (Shard expiredShard : expiredShards) { boolean successorFound = false; for (Shard shard : allShards) { if (Objects.equals(expiredShard.getShardId(), shard.getParentShardId())) { nextShards.add(shard); successorFound = true; } else if (Objects.equals(expiredShard.getShardId(), shard.getAdjacentParentShardId())) { successorFound = true; } } if (!successorFound) { // This can potentially happen during split/merge operation. Newly created shards might be // not listed in the allShards list and their predecessor is already considered expired. // Retrying should solve the issue. throw new IllegalStateException("No successors were found for shard: " + expiredShard); } } return nextShards; }
/** * Helper method to return all the open shards for a stream. * Note: Package level access only for testing purposes. * * @param allShards All shards returved via DescribeStream. We assume this to represent a consistent shard list. * @return List of open shards (shards at the tip of the stream) - may include shards that are not yet active. */ static List<Shard> getOpenShards(List<Shard> allShards) { List<Shard> openShards = new ArrayList<Shard>(); for (Shard shard : allShards) { String endingSequenceNumber = shard.getSequenceNumberRange().getEndingSequenceNumber(); if (endingSequenceNumber == null) { openShards.add(shard); LOG.debug("Found open shard: " + shard.getShardId()); } } return openShards; }
/** * Finds the initial set of shards (the oldest ones). These shards do not have their parents in * the shard list. */ private Set<Shard> findInitialShardsWithoutParents(String streamName, List<Shard> allShards) { Set<String> shardIds = new HashSet<>(); for (Shard shard : allShards) { shardIds.add(shard.getShardId()); } LOGGER.info("Stream {} has following shards: {}", streamName, shardIds); Set<Shard> shardsWithoutParents = new HashSet<>(); for (Shard shard : allShards) { if (!shardIds.contains(shard.getParentShardId())) { shardsWithoutParents.add(shard); } } return shardsWithoutParents; }
Set<String> childShardIds) throws KinesisClientLibIOException { BigInteger startingHashKeyOfClosedShard = new BigInteger(closedShard.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKeyOfClosedShard = new BigInteger(closedShard.getHashKeyRange().getEndingHashKey()); BigInteger minStartingHashKeyOfChildren = null; BigInteger maxEndingHashKeyOfChildren = null; BigInteger startingHashKey = new BigInteger(childShard.getHashKeyRange().getStartingHashKey()); if ((minStartingHashKeyOfChildren == null) || (startingHashKey.compareTo(minStartingHashKeyOfChildren) < 0)) { minStartingHashKeyOfChildren = startingHashKey; BigInteger endingHashKey = new BigInteger(childShard.getHashKeyRange().getEndingHashKey()); if ((maxEndingHashKeyOfChildren == null) || (endingHashKey.compareTo(maxEndingHashKeyOfChildren) > 0)) { || (maxEndingHashKeyOfChildren.compareTo(endingHashKeyOfClosedShard) < 0)) { throw new KinesisClientLibIOException("Incomplete shard list: hash key range of shard " + closedShard.getShardId() + " is not covered by its child shards.");
/** * Helper method to get parent shardIds of the current shard - includes the parent shardIds if: * a/ they are not null * b/ if they exist in the current shard map (i.e. haven't expired) * * @param shard Will return parents of this shard * @param shardIdToShardMapOfAllKinesisShards ShardId->Shard map containing all shards obtained via DescribeStream. * @return Set of parentShardIds */ static Set<String> getParentShardIds(Shard shard, Map<String, Shard> shardIdToShardMapOfAllKinesisShards) { Set<String> parentShardIds = new HashSet<String>(2); String parentShardId = shard.getParentShardId(); if ((parentShardId != null) && shardIdToShardMapOfAllKinesisShards.containsKey(parentShardId)) { parentShardIds.add(parentShardId); } String adjacentParentShardId = shard.getAdjacentParentShardId(); if ((adjacentParentShardId != null) && shardIdToShardMapOfAllKinesisShards.containsKey(adjacentParentShardId)) { parentShardIds.add(adjacentParentShardId); } return parentShardIds; }
boolean isShardClosed() { return shard.getSequenceNumberRange() != null && shard.getSequenceNumberRange().getEndingSequenceNumber() != null; }
public int compare(Shard o1, Shard o2) { return new BigInteger(o1.getHashKeyRange().getStartingHashKey()) .compareTo(new BigInteger(o2.getHashKeyRange() .getStartingHashKey())); } });
private Shard createClosedShard(String shardId) { Shard shard = new Shard().withShardId(shardId); activeAtPoint(shard, ShardIteratorType.TRIM_HORIZON); expiredAtPoint(shard, ShardIteratorType.LATEST); return shard; }
/** * Adds descendants only to closed leaf nodes in order to ensure all leaf nodes in * the graph are open. * @param shards list of shards obtained from DescribeStream call. */ private void addToClosedLeafNodes(List<Shard> shards) { if (null == shards) { return; } if (LOG.isDebugEnabled()) { LOG.debug(String.format("Attempting to resolve inconsistencies in the graph with the following shards: \n %s", String.join(", ", shards.stream().map(Shard::getShardId).collect(Collectors.toList())))); } for (Shard shard : shards) { final String parentShardId = shard.getParentShardId(); if (null != parentShardId && closedLeafNodeIds.contains(parentShardId)) { ShardNode shardNode = addNode(shard); closedLeafNodeIds.remove(parentShardId); if (shardNode.isShardClosed()) { closedLeafNodeIds.add(shardNode.getShardId()); } } } updateLastFetchedShardId(shards); }
.withStreamStatus(StreamStatus.ACTIVE) .withHasMoreShards(false) .withShards(new Shard() .withShardId("closedShard") .withSequenceNumberRange(new SequenceNumberRange() .withEndingSequenceNumber("1")))));
List<Shard> getShardsForStream (String stream) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(stream); List<Shard> shards = new ArrayList<>(); String exclusiveStartShardId = null; do { describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while ( exclusiveStartShardId != null ); LOG.info("Number of shards for stream " + stream + " are " + shards.size()); return shards; }
private String addTruncatedShardList(final Map<String, ShardInfo> spoutShards, final List<Shard> streamShards) { String currShard = ""; for (Shard s : streamShards) { currShard = s.getShardId(); spoutShards.put(s.getShardId(), new ShardInfo(s.getShardId())); if (s.getParentShardId() != null && s.getAdjacentParentShardId() != null) { // It's a merge. Set both parents of the merge to merge into this shard. ShardInfo parentShardInfo = spoutShards.get(s.getParentShardId()); ShardInfo adjacentParentShardInfo = spoutShards.get(s.getAdjacentParentShardId()); if ((parentShardInfo != null) && (adjacentParentShardInfo != null)) { parentShardInfo.setMergesInto(s.getShardId()); adjacentParentShardInfo.setMergesInto(s.getShardId()); } } else if (s.getParentShardId() != null) { // It's a split. Add the current shard to the split list of its parent. ShardInfo parentShardInfo = spoutShards.get(s.getParentShardId()); if (parentShardInfo != null) { parentShardInfo.addSplitsInto(s.getShardId()); } } } return currShard; } }
try { List<Record> records = KinesisUtil.getInstance().getRecords(streamName, recordsLimit, shard.getShardId(), getIteratorType(shard.getShardId()), shardPosition.get(shard.getShardId())); if (shard.getSequenceNumberRange().getEndingSequenceNumber() != null) { closedShards.add(shard); break; for (Record rc : records) { seqNo = rc.getSequenceNumber(); putRecord(shd.getShardId(), rc); shardPosition.put(shard.getShardId(), seqNo);
private ShardNode addNode(Shard shard) { final ShardNode shardNode = new ShardNode(shard); nodes.put(shardNode.getShardId(), shardNode); // if the node is closed, add it to the closed leaf node set. // once its child appears, this node will be removed from the set. if (shardNode.isShardClosed()) { closedLeafNodeIds.add(shardNode.getShardId()); } final String parentShardID = shard.getParentShardId(); // Ensure nodes contains the parent shard, since older shards are trimmed and we will see nodes whose // parent shards are not in the graph. if (null != parentShardID && nodes.containsKey(parentShardID)) { final ShardNode parentNode = nodes.get(parentShardID); parentNode.addDescendant(shard.getShardId()); closedLeafNodeIds.remove(parentShardID); } return shardNode; }
if (!info.getShard().getShardId().equals(this.shard.getShardId())) { if (info.getShard().getHashKeyRange().getStartingHashKey().equals(targetHash.toString())) { higherShard = new ShardHashInfo(this.streamName, info.getShard()); break;
String shardId = entry.getKey(); Shard shard = entry.getValue(); String parentShardId = shard.getParentShardId(); if ((parentShardId != null) && (shardIdToShardMap.containsKey(parentShardId))) { Set<String> childShardIds = shardIdToChildShardIdsMap.get(parentShardId); String adjacentParentShardId = shard.getAdjacentParentShardId(); if ((adjacentParentShardId != null) && (shardIdToShardMap.containsKey(adjacentParentShardId))) { Set<String> childShardIds = shardIdToChildShardIdsMap.get(adjacentParentShardId);
boolean isShardClosed() { return shard.getSequenceNumberRange() != null && shard.getSequenceNumberRange().getEndingSequenceNumber() != null; }