/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: ").append(getShardId()).append(","); if (getParentShardId() != null) sb.append("ParentShardId: ").append(getParentShardId()).append(","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: ").append(getAdjacentParentShardId()).append(","); if (getHashKeyRange() != null) sb.append("HashKeyRange: ").append(getHashKeyRange()).append(","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: ").append(getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
/** * Marshall the given parameter object. */ public void marshall(Shard shard, ProtocolMarshaller protocolMarshaller) { if (shard == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(shard.getShardId(), SHARDID_BINDING); protocolMarshaller.marshall(shard.getParentShardId(), PARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getAdjacentParentShardId(), ADJACENTPARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getHashKeyRange(), HASHKEYRANGE_BINDING); protocolMarshaller.marshall(shard.getSequenceNumberRange(), SEQUENCENUMBERRANGE_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
if (other.getAdjacentParentShardId() != null && other.getAdjacentParentShardId().equals(this.getAdjacentParentShardId()) == false) return false; if (other.getHashKeyRange() == null ^ this.getHashKeyRange() == null) return false; if (other.getHashKeyRange() != null && other.getHashKeyRange().equals(this.getHashKeyRange()) == false) return false; if (other.getSequenceNumberRange() == null ^ this.getSequenceNumberRange() == null)
/** * Returns a string representation of this object; useful for testing and * debugging. * * @return A string representation of this object. * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: " + getShardId() + ","); if (getParentShardId() != null) sb.append("ParentShardId: " + getParentShardId() + ","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: " + getAdjacentParentShardId() + ","); if (getHashKeyRange() != null) sb.append("HashKeyRange: " + getHashKeyRange() + ","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: " + getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
&& other.getAdjacentParentShardId().equals(this.getAdjacentParentShardId()) == false) return false; if (other.getHashKeyRange() == null ^ this.getHashKeyRange() == null) return false; if (other.getHashKeyRange() != null && other.getHashKeyRange().equals(this.getHashKeyRange()) == false) return false; if (other.getSequenceNumberRange() == null ^ this.getSequenceNumberRange() == null)
public void marshall(Shard shard, AwsJsonWriter jsonWriter) throws Exception { jsonWriter.beginObject(); if (shard.getShardId() != null) { String shardId = shard.getShardId(); jsonWriter.name("ShardId"); jsonWriter.value(shardId); } if (shard.getParentShardId() != null) { String parentShardId = shard.getParentShardId(); jsonWriter.name("ParentShardId"); jsonWriter.value(parentShardId); } if (shard.getAdjacentParentShardId() != null) { String adjacentParentShardId = shard.getAdjacentParentShardId(); jsonWriter.name("AdjacentParentShardId"); jsonWriter.value(adjacentParentShardId); } if (shard.getHashKeyRange() != null) { HashKeyRange hashKeyRange = shard.getHashKeyRange(); jsonWriter.name("HashKeyRange"); HashKeyRangeJsonMarshaller.getInstance().marshall(hashKeyRange, jsonWriter); } if (shard.getSequenceNumberRange() != null) { SequenceNumberRange sequenceNumberRange = shard.getSequenceNumberRange(); jsonWriter.name("SequenceNumberRange"); SequenceNumberRangeJsonMarshaller.getInstance().marshall(sequenceNumberRange, jsonWriter); } jsonWriter.endObject(); }
public int compare(Shard o1, Shard o2) { return new BigInteger(o1.getHashKeyRange().getStartingHashKey()) .compareTo(new BigInteger(o2.getHashKeyRange() .getStartingHashKey())); } });
private static final int compareShardsByStartHash(Shard o1, Shard o2) { return new BigInteger(o1.getHashKeyRange().getStartingHashKey()) .compareTo(new BigInteger(o2.getHashKeyRange().getStartingHashKey())); }
public ShardHashInfo(String streamName, Shard shard) { // prevent constructing a null object if (streamName == null || shard == null) { throw new ExceptionInInitializerError("Stream Name & Shard Required"); } this.shard = shard; this.streamName = streamName; this.endHash = new BigInteger(shard.getHashKeyRange().getEndingHashKey()); this.startHash = new BigInteger(shard.getHashKeyRange().getStartingHashKey()); this.hashWidth = getWidth(this.startHash, this.endHash); this.pctOfKeyspace = getPctOfKeyspace(this.hashWidth); }
public AdjacentShards(String streamName, ShardHashInfo lower, ShardHashInfo higher) throws Exception { // ensure that the shards are adjacent if (!new BigInteger(higher.getShard().getHashKeyRange().getStartingHashKey()) .subtract(new BigInteger(lower.getShard().getHashKeyRange().getEndingHashKey())) .equals(new BigInteger("1"))) { throw new Exception("Shards are not Adjacent"); } this.streamName = streamName; this.lowerShard = lower; this.higherShard = higher; }
Set<String> childShardIds) throws KinesisClientLibIOException { BigInteger startingHashKeyOfClosedShard = new BigInteger(closedShard.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKeyOfClosedShard = new BigInteger(closedShard.getHashKeyRange().getEndingHashKey()); BigInteger minStartingHashKeyOfChildren = null; BigInteger maxEndingHashKeyOfChildren = null; BigInteger startingHashKey = new BigInteger(childShard.getHashKeyRange().getStartingHashKey()); if ((minStartingHashKeyOfChildren == null) || (startingHashKey.compareTo(minStartingHashKeyOfChildren) < 0)) { minStartingHashKeyOfChildren = startingHashKey; BigInteger endingHashKey = new BigInteger(childShard.getHashKeyRange().getEndingHashKey()); if ((maxEndingHashKeyOfChildren == null) || (endingHashKey.compareTo(maxEndingHashKeyOfChildren) > 0)) {
/** * Determines whether to deaggregate the given records, and if they are KPL records dispatches them to deaggregation * * @param records * the records to deaggregate is deaggregation is required. * @return returns either the deaggregated records, or the original records */ @SuppressWarnings("unchecked") private List<Record> deaggregateRecords(List<Record> records) { // We deaggregate if and only if we got actual Kinesis records, i.e. // not instances of some subclass thereof. if (!records.isEmpty() && records.get(0).getClass().equals(Record.class)) { if (this.shard != null) { return (List<Record>) (List<?>) UserRecord.deaggregate(records, new BigInteger(this.shard.getHashKeyRange().getStartingHashKey()), new BigInteger(this.shard.getHashKeyRange().getEndingHashKey())); } else { return (List<Record>) (List<?>) UserRecord.deaggregate(records); } } return records; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: ").append(getShardId()).append(","); if (getParentShardId() != null) sb.append("ParentShardId: ").append(getParentShardId()).append(","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: ").append(getAdjacentParentShardId()).append(","); if (getHashKeyRange() != null) sb.append("HashKeyRange: ").append(getHashKeyRange()).append(","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: ").append(getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
private void sentWatermarkToShards() { try { //refresh the list of available shards, if current state is too old if (System.currentTimeMillis() - lastShardRefreshTime >= SHARD_REFRESH_MILLIES) { refreshShards(); lastShardRefreshTime = System.currentTimeMillis(); } //send a watermark to every shard of the Kinesis stream shards.parallelStream() .map(shard -> new PutRecordRequest() .withStreamName(streamName) .withData(new WatermarkEvent(currentWatermark).toByteBuffer()) .withPartitionKey("23") .withExplicitHashKey(shard.getHashKeyRange().getStartingHashKey())) .map(kinesisClient::putRecord) .forEach(putRecordResult -> LOG.trace("send watermark {} to shard {}", new DateTime(currentWatermark), putRecordResult.getShardId())); LOG.debug("send watermark {}", new DateTime(currentWatermark)); } catch (LimitExceededException | ProvisionedThroughputExceededException e) { //if any request is throttled, just wait for the next iteration to submit another watermark LOG.warn("skipping watermark due to limit/throughput exceeded exception"); } }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
/** * Marshall the given parameter object. */ public void marshall(Shard shard, ProtocolMarshaller protocolMarshaller) { if (shard == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(shard.getShardId(), SHARDID_BINDING); protocolMarshaller.marshall(shard.getParentShardId(), PARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getAdjacentParentShardId(), ADJACENTPARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getHashKeyRange(), HASHKEYRANGE_BINDING); protocolMarshaller.marshall(shard.getSequenceNumberRange(), SEQUENCENUMBERRANGE_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
if (other.getAdjacentParentShardId() != null && other.getAdjacentParentShardId().equals(this.getAdjacentParentShardId()) == false) return false; if (other.getHashKeyRange() == null ^ this.getHashKeyRange() == null) return false; if (other.getHashKeyRange() != null && other.getHashKeyRange().equals(this.getHashKeyRange()) == false) return false; if (other.getSequenceNumberRange() == null ^ this.getSequenceNumberRange() == null)
if (info.getShard().getHashKeyRange().getStartingHashKey().equals(targetHash.toString())) { higherShard = new ShardHashInfo(this.streamName, info.getShard()); break;