/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: ").append(getShardId()).append(","); if (getParentShardId() != null) sb.append("ParentShardId: ").append(getParentShardId()).append(","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: ").append(getAdjacentParentShardId()).append(","); if (getHashKeyRange() != null) sb.append("HashKeyRange: ").append(getHashKeyRange()).append(","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: ").append(getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
/** * Marshall the given parameter object. */ public void marshall(Shard shard, ProtocolMarshaller protocolMarshaller) { if (shard == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(shard.getShardId(), SHARDID_BINDING); protocolMarshaller.marshall(shard.getParentShardId(), PARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getAdjacentParentShardId(), ADJACENTPARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getHashKeyRange(), HASHKEYRANGE_BINDING); protocolMarshaller.marshall(shard.getSequenceNumberRange(), SEQUENCENUMBERRANGE_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
if (other.getShardId() != null && other.getShardId().equals(this.getShardId()) == false) return false; if (other.getParentShardId() == null ^ this.getParentShardId() == null) return false; if (other.getParentShardId() != null && other.getParentShardId().equals(this.getParentShardId()) == false) return false; if (other.getAdjacentParentShardId() == null ^ this.getAdjacentParentShardId() == null)
/** * Returns a string representation of this object; useful for testing and * debugging. * * @return A string representation of this object. * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: " + getShardId() + ","); if (getParentShardId() != null) sb.append("ParentShardId: " + getParentShardId() + ","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: " + getAdjacentParentShardId() + ","); if (getHashKeyRange() != null) sb.append("HashKeyRange: " + getHashKeyRange() + ","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: " + getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
if (other.getShardId() != null && other.getShardId().equals(this.getShardId()) == false) return false; if (other.getParentShardId() == null ^ this.getParentShardId() == null) return false; if (other.getParentShardId() != null && other.getParentShardId().equals(this.getParentShardId()) == false) return false; if (other.getAdjacentParentShardId() == null ^ this.getAdjacentParentShardId() == null)
public void marshall(Shard shard, AwsJsonWriter jsonWriter) throws Exception { jsonWriter.beginObject(); if (shard.getShardId() != null) { String shardId = shard.getShardId(); jsonWriter.name("ShardId"); jsonWriter.value(shardId); } if (shard.getParentShardId() != null) { String parentShardId = shard.getParentShardId(); jsonWriter.name("ParentShardId"); jsonWriter.value(parentShardId); } if (shard.getAdjacentParentShardId() != null) { String adjacentParentShardId = shard.getAdjacentParentShardId(); jsonWriter.name("AdjacentParentShardId"); jsonWriter.value(adjacentParentShardId); } if (shard.getHashKeyRange() != null) { HashKeyRange hashKeyRange = shard.getHashKeyRange(); jsonWriter.name("HashKeyRange"); HashKeyRangeJsonMarshaller.getInstance().marshall(hashKeyRange, jsonWriter); } if (shard.getSequenceNumberRange() != null) { SequenceNumberRange sequenceNumberRange = shard.getSequenceNumberRange(); jsonWriter.name("SequenceNumberRange"); SequenceNumberRangeJsonMarshaller.getInstance().marshall(sequenceNumberRange, jsonWriter); } jsonWriter.endObject(); }
/** * Finds the initial set of shards (the oldest ones). These shards do not have their parents in * the shard list. */ private Set<Shard> findInitialShardsWithoutParents(String streamName, List<Shard> allShards) { Set<String> shardIds = new HashSet<>(); for (Shard shard : allShards) { shardIds.add(shard.getShardId()); } LOGGER.info("Stream {} has following shards: {}", streamName, shardIds); Set<Shard> shardsWithoutParents = new HashSet<>(); for (Shard shard : allShards) { if (!shardIds.contains(shard.getParentShardId())) { shardsWithoutParents.add(shard); } } return shardsWithoutParents; }
/** * Helper method to get parent shardIds of the current shard - includes the parent shardIds if: * a/ they are not null * b/ if they exist in the current shard map (i.e. haven't expired) * * @param shard Will return parents of this shard * @param shardIdToShardMapOfAllKinesisShards ShardId->Shard map containing all shards obtained via DescribeStream. * @return Set of parentShardIds */ static Set<String> getParentShardIds(Shard shard, Map<String, Shard> shardIdToShardMapOfAllKinesisShards) { Set<String> parentShardIds = new HashSet<String>(2); String parentShardId = shard.getParentShardId(); if ((parentShardId != null) && shardIdToShardMapOfAllKinesisShards.containsKey(parentShardId)) { parentShardIds.add(parentShardId); } String adjacentParentShardId = shard.getAdjacentParentShardId(); if ((adjacentParentShardId != null) && shardIdToShardMapOfAllKinesisShards.containsKey(adjacentParentShardId)) { parentShardIds.add(adjacentParentShardId); } return parentShardIds; }
private Set<Shard> findNextShards(List<Shard> allShards, Set<Shard> expiredShards) { Set<Shard> nextShards = new HashSet<>(); for (Shard expiredShard : expiredShards) { boolean successorFound = false; for (Shard shard : allShards) { if (Objects.equals(expiredShard.getShardId(), shard.getParentShardId())) { nextShards.add(shard); successorFound = true; } else if (Objects.equals(expiredShard.getShardId(), shard.getAdjacentParentShardId())) { successorFound = true; } } if (!successorFound) { // This can potentially happen during split/merge operation. Newly created shards might be // not listed in the allShards list and their predecessor is already considered expired. // Retrying should solve the issue. throw new IllegalStateException("No successors were found for shard: " + expiredShard); } } return nextShards; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: ").append(getShardId()).append(","); if (getParentShardId() != null) sb.append("ParentShardId: ").append(getParentShardId()).append(","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: ").append(getAdjacentParentShardId()).append(","); if (getHashKeyRange() != null) sb.append("HashKeyRange: ").append(getHashKeyRange()).append(","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: ").append(getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
private ShardNode addNode(Shard shard) { final ShardNode shardNode = new ShardNode(shard); nodes.put(shardNode.getShardId(), shardNode); // if the node is closed, add it to the closed leaf node set. // once its child appears, this node will be removed from the set. if (shardNode.isShardClosed()) { closedLeafNodeIds.add(shardNode.getShardId()); } final String parentShardID = shard.getParentShardId(); // Ensure nodes contains the parent shard, since older shards are trimmed and we will see nodes whose // parent shards are not in the graph. if (null != parentShardID && nodes.containsKey(parentShardID)) { final ShardNode parentNode = nodes.get(parentShardID); parentNode.addDescendant(shard.getShardId()); closedLeafNodeIds.remove(parentShardID); } return shardNode; }
private String addTruncatedShardList(final Map<String, ShardInfo> spoutShards, final List<Shard> streamShards) { String currShard = ""; for (Shard s : streamShards) { currShard = s.getShardId(); spoutShards.put(s.getShardId(), new ShardInfo(s.getShardId())); if (s.getParentShardId() != null && s.getAdjacentParentShardId() != null) { // It's a merge. Set both parents of the merge to merge into this shard. ShardInfo parentShardInfo = spoutShards.get(s.getParentShardId()); ShardInfo adjacentParentShardInfo = spoutShards.get(s.getAdjacentParentShardId()); if ((parentShardInfo != null) && (adjacentParentShardInfo != null)) { parentShardInfo.setMergesInto(s.getShardId()); adjacentParentShardInfo.setMergesInto(s.getShardId()); } } else if (s.getParentShardId() != null) { // It's a split. Add the current shard to the split list of its parent. ShardInfo parentShardInfo = spoutShards.get(s.getParentShardId()); if (parentShardInfo != null) { parentShardInfo.addSplitsInto(s.getShardId()); } } } return currShard; } }
private ShardNode addNode(Shard shard) { final ShardNode shardNode = new ShardNode(shard); nodes.put(shardNode.getShardId(), shardNode); // if the node is closed, add it to the closed leaf node set. // once its child appears, this node will be removed from the set. if (shardNode.isShardClosed()) { closedLeafNodeIds.add(shardNode.getShardId()); } final String parentShardID = shard.getParentShardId(); // Ensure nodes contains the parent shard, since older shards are trimmed and we will see nodes whose // parent shards are not in the graph. if (null != parentShardID && nodes.containsKey(parentShardID)) { final ShardNode parentNode = nodes.get(parentShardID); parentNode.addDescendant(shard.getShardId()); closedLeafNodeIds.remove(parentShardID); } return shardNode; }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
/** * Helper method to create a new KinesisClientLease POJO for a shard. * Note: Package level access only for testing purposes * * @param shard * @return */ static KinesisClientLease newKCLLease(Shard shard) { KinesisClientLease newLease = new KinesisClientLease(); newLease.setLeaseKey(shard.getShardId()); List<String> parentShardIds = new ArrayList<String>(2); if (shard.getParentShardId() != null) { parentShardIds.add(shard.getParentShardId()); } if (shard.getAdjacentParentShardId() != null) { parentShardIds.add(shard.getAdjacentParentShardId()); } newLease.setParentShardIds(parentShardIds); newLease.setOwnerSwitchesSinceCheckpoint(0L); return newLease; }
List<ShardRecordsIterator> findSuccessiveShardRecordIterators() throws TransientKinesisException { List<Shard> shards = kinesis.listShards(streamName); List<ShardRecordsIterator> successiveShardRecordIterators = new ArrayList<>(); for (Shard shard : shards) { if (shardId.equals(shard.getParentShardId())) { ShardCheckpoint shardCheckpoint = new ShardCheckpoint( streamName, shard.getShardId(), new StartingPoint(InitialPositionInStream.TRIM_HORIZON)); successiveShardRecordIterators.add(new ShardRecordsIterator(shardCheckpoint, kinesis)); } } return successiveShardRecordIterators; } }
/** * Marshall the given parameter object. */ public void marshall(Shard shard, ProtocolMarshaller protocolMarshaller) { if (shard == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(shard.getShardId(), SHARDID_BINDING); protocolMarshaller.marshall(shard.getParentShardId(), PARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getAdjacentParentShardId(), ADJACENTPARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getHashKeyRange(), HASHKEYRANGE_BINDING); protocolMarshaller.marshall(shard.getSequenceNumberRange(), SEQUENCENUMBERRANGE_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
/** * Merge these two Shards and return the result Shard * * @param kinesisClient * @return * @throws Exception */ protected ShardHashInfo doMerge(AmazonKinesisClient kinesisClient, String currentHighestShardId) throws Exception { StreamScalingUtils.mergeShards(kinesisClient, streamName, this.lowerShard, this.higherShard, true); Map<String, ShardHashInfo> openShards = StreamScalingUtils.getOpenShards(kinesisClient, streamName, currentHighestShardId); for (ShardHashInfo info : openShards.values()) { if (lowerShard.getShardId().equals(info.getShard().getParentShardId()) && higherShard.getShardId().equals(info.getShard().getAdjacentParentShardId())) { return new ShardHashInfo(streamName, info.getShard()); } } throw new Exception(String.format("Unable resolve new created Shard for parents %s and %s", lowerShard.getShardId(), higherShard.getShardId())); } }