@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getSequenceNumber() == null) ? 0 : getSequenceNumber().hashCode()); hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getErrorCode() == null) ? 0 : getErrorCode().hashCode()); hashCode = prime * hashCode + ((getErrorMessage() == null) ? 0 : getErrorMessage().hashCode()); return hashCode; }
public PutRecordsResultEntry unmarshall(JsonUnmarshallerContext context) throws Exception { PutRecordsResultEntry putRecordsResultEntry = new PutRecordsResultEntry(); if (context.testExpression("SequenceNumber", targetDepth)) { context.nextToken(); putRecordsResultEntry.setSequenceNumber(context.getUnmarshaller(String.class).unmarshall(context)); putRecordsResultEntry.setShardId(context.getUnmarshaller(String.class).unmarshall(context)); putRecordsResultEntry.setErrorCode(context.getUnmarshaller(String.class).unmarshall(context)); putRecordsResultEntry.setErrorMessage(context.getUnmarshaller(String.class).unmarshall(context));
for (final PutRecordsResultEntry responseEntry : result.getRecords()) { final PutRecordsRequestEntry record = requestRecords.get(index); if (responseEntry.getErrorCode() == null) { sentRecords.add(index); totalBytesSent += record.getData().limit(); } else { logger.trace("{}:{} Record {} returned error code {}: {}", flow.getId(), buffer, index, responseEntry.getErrorCode(), responseEntry.getErrorMessage()); errors.add(responseEntry.getErrorCode());
.mapToObj(i -> { PutRecordsRequestEntry e = entries.get(i); String errorCode = results.get(i).getErrorCode(); int n = recordAttempts.getOrDefault(e, 1) + 1;
result.setFailedRecordCount(5); for (int i = 0; i < count; i++) { PutRecordsResultEntry entry = new PutRecordsResultEntry(); entry.setSequenceNumber("record_id_" + i); } else { entry.setErrorCode("ServiceUnavailable");
public Observable<PutRecordsResultEntry> play(LocalDateTime start, @Nullable LocalDateTime end) { return playableObjects(start, end) .onBackpressureBuffer() .observeOn(Schedulers.io()) .flatMap(this::objectToPayloads) .map(ByteBuffer::wrap) .lift(new OperatorBufferKinesisBatch(MAX_KINESIS_BATCH_SIZE, MAX_KINESIS_BATCH_WEIGHT)) .onBackpressureBuffer() .map(byteBuffers -> byteBuffers.stream() .map(buffer -> new PutRecordsRequestEntry() .withData(buffer) .withPartitionKey(UUID.randomUUID().toString())) .collect(toList())) .map(entries -> new PutRecordsRequest() .withStreamName(vcrConfiguration.targetStream) .withRecords(entries)) .observeOn(Schedulers.io()) .flatMap(putRecordsRequest -> Observable.create((Observable.OnSubscribe<List<PutRecordsResult>>) os -> { os.onStart(); kinesisWriter.submit(() -> { os.onNext(putWithRetry(putRecordsRequest).orElse(Collections.<PutRecordsResult>emptyList())); os.onCompleted(); }); })) .flatMap(Observable::from) .flatMap(putRecordsResult -> Observable.from(putRecordsResult.getRecords())) .doOnNext(result -> LOGGER.debug("Wrote record. Seq {}, shard {}", result.getSequenceNumber(), result.getShardId())); }
@Test(expected = AmazonClientException.class) public void testSendBatchException() { String streamName = "stream"; int count = 10; // create an ok result PutRecordsResult result = new PutRecordsResult(); List<PutRecordsResultEntry> entries = new ArrayList<PutRecordsResultEntry>(); result.setFailedRecordCount(0); for (int i = 0; i < count; i++) { PutRecordsResultEntry entry = new PutRecordsResultEntry(); entry.setSequenceNumber("record_id_" + i); entries.add(entry); } result.setRecords(entries); // create data List<byte[]> data = new ArrayList<byte[]>(); for (int i = 0; i < count; i++) { data.add(("record" + i).getBytes(StringUtils.UTF8)); } Mockito.when(client.putRecords(any(PutRecordsRequest.class))).thenThrow( new InvalidArgumentException("invalid argument")); sender.sendBatch(streamName, data); }
public void storeBatchInline(List<Event> events, int offset, int limit) { PutRecordsRequestEntry[] records = new PutRecordsRequestEntry[limit]; for (int i = 0; i < limit; i++) { Event event = events.get(offset + i); PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry() .withData(getBuffer(event)) .withPartitionKey(event.project() + "|" + event.collection()); records[i] = putRecordsRequestEntry; } try { PutRecordsResult putRecordsResult = kinesis.putRecords(new PutRecordsRequest() .withRecords(records) .withStreamName(config.getEventStoreStreamName())); if (putRecordsResult.getFailedRecordCount() > 0) { for (PutRecordsResultEntry resultEntry : putRecordsResult.getRecords()) { resultEntry.getErrorMessage(); } } } catch (ResourceNotFoundException e) { try { createAndWaitForStreamToBecomeAvailable(kinesis, config.getEventStoreStreamName(), 1); } catch (Exception e1) { throw new RuntimeException("Couldn't send event to Amazon Kinesis", e); } } }
/** * <p> * The sequence number for an individual record result. * </p> * * @param sequenceNumber * The sequence number for an individual record result. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsResultEntry withSequenceNumber(String sequenceNumber) { setSequenceNumber(sequenceNumber); return this; }
/** * <p> * The error code for an individual record result. <code>ErrorCodes</code> can be either * <code>ProvisionedThroughputExceededException</code> or <code>InternalFailure</code>. * </p> * * @param errorCode * The error code for an individual record result. <code>ErrorCodes</code> can be either * <code>ProvisionedThroughputExceededException</code> or <code>InternalFailure</code>. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsResultEntry withErrorCode(String errorCode) { setErrorCode(errorCode); return this; }
if (recordsResponse.get(i).getErrorCode() != null) { failedRecordIndexes[idx++] = i; errors.compute(recordsResponse.get(i).getErrorMessage(), (k, v) -> v == null ? 1 : v++);
@Override public List<byte[]> sendBatch(String streamName, List<byte[]> data) { if (data == null || data.isEmpty()) { return Collections.emptyList(); } final PutRecordsRequest request = new PutRecordsRequest(); request.setStreamName(streamName); final List<PutRecordsRequestEntry> records = new ArrayList<PutRecordsRequestEntry>(data.size()); for (final byte[] d : data) { final String partKey = StringUtils.isBlank(this.partitionKey) ? UUID.randomUUID().toString() : this.partitionKey; final PutRecordsRequestEntry r = new PutRecordsRequestEntry(); r.setData(ByteBuffer.wrap(d)); r.setPartitionKey(partKey); records.add(r); } request.setRecords(records); request.getRequestClientOptions().appendUserAgent(userAgent); final PutRecordsResult result = client.putRecords(request); final int size = result.getRecords().size(); final List<byte[]> failures = new ArrayList<byte[]>(result.getFailedRecordCount()); for (int i = 0; i < size; i++) { if (result.getRecords().get(i).getErrorCode() != null) { // always retry failed record failures.add(data.get(i)); } } return failures; }
result.setFailedRecordCount(0); for (int i = 0; i < count; i++) { PutRecordsResultEntry entry = new PutRecordsResultEntry(); entry.setSequenceNumber("record_id_" + i); entries.add(entry);
/** * <p> * The sequence number for an individual record result. * </p> * * @param sequenceNumber * The sequence number for an individual record result. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsResultEntry withSequenceNumber(String sequenceNumber) { setSequenceNumber(sequenceNumber); return this; }
/** * <p> * The error code for an individual record result. <code>ErrorCodes</code> can be either * <code>ProvisionedThroughputExceededException</code> or <code>InternalFailure</code>. * </p> * * @param errorCode * The error code for an individual record result. <code>ErrorCodes</code> can be either * <code>ProvisionedThroughputExceededException</code> or <code>InternalFailure</code>. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsResultEntry withErrorCode(String errorCode) { setErrorCode(errorCode); return this; }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof PutRecordsResultEntry == false) return false; PutRecordsResultEntry other = (PutRecordsResultEntry) obj; if (other.getSequenceNumber() == null ^ this.getSequenceNumber() == null) return false; if (other.getSequenceNumber() != null && other.getSequenceNumber().equals(this.getSequenceNumber()) == false) return false; if (other.getShardId() == null ^ this.getShardId() == null) return false; if (other.getShardId() != null && other.getShardId().equals(this.getShardId()) == false) return false; if (other.getErrorCode() == null ^ this.getErrorCode() == null) return false; if (other.getErrorCode() != null && other.getErrorCode().equals(this.getErrorCode()) == false) return false; if (other.getErrorMessage() == null ^ this.getErrorMessage() == null) return false; if (other.getErrorMessage() != null && other.getErrorMessage().equals(this.getErrorMessage()) == false) return false; return true; }
public PutRecordsResultEntry unmarshall(JsonUnmarshallerContext context) throws Exception { AwsJsonReader reader = context.getReader(); if (!reader.isContainer()) { reader.skipValue(); return null; } PutRecordsResultEntry putRecordsResultEntry = new PutRecordsResultEntry(); reader.beginObject(); while (reader.hasNext()) { String name = reader.nextName(); if (name.equals("SequenceNumber")) { putRecordsResultEntry.setSequenceNumber(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else if (name.equals("ShardId")) { putRecordsResultEntry.setShardId(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else if (name.equals("ErrorCode")) { putRecordsResultEntry.setErrorCode(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else if (name.equals("ErrorMessage")) { putRecordsResultEntry.setErrorMessage(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else { reader.skipValue(); } } reader.endObject(); return putRecordsResultEntry; }
PutRecordsResultEntry resultEntry = putRecordsResult.getRecords().get(i); PutRecordsRequestEntry requestEntry = putRecordsRequest.getRecords().get(i); if (resultEntry.getErrorCode() != null) { entriesForRetry.add(requestEntry);
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getSequenceNumber() != null) sb.append("SequenceNumber: ").append(getSequenceNumber()).append(","); if (getShardId() != null) sb.append("ShardId: ").append(getShardId()).append(","); if (getErrorCode() != null) sb.append("ErrorCode: ").append(getErrorCode()).append(","); if (getErrorMessage() != null) sb.append("ErrorMessage: ").append(getErrorMessage()); sb.append("}"); return sb.toString(); }
public PutRecordsResultEntry unmarshall(JsonUnmarshallerContext context) throws Exception { PutRecordsResultEntry putRecordsResultEntry = new PutRecordsResultEntry(); if (context.testExpression("SequenceNumber", targetDepth)) { context.nextToken(); putRecordsResultEntry.setSequenceNumber(context.getUnmarshaller(String.class).unmarshall(context)); putRecordsResultEntry.setShardId(context.getUnmarshaller(String.class).unmarshall(context)); putRecordsResultEntry.setErrorCode(context.getUnmarshaller(String.class).unmarshall(context)); putRecordsResultEntry.setErrorMessage(context.getUnmarshaller(String.class).unmarshall(context));