PutRecordsRequest putRecordRequest = new PutRecordsRequest(); putRecordRequest.setStreamName(streamName); putRecordRequest.setRecords(records); PutRecordsResult results = client.putRecords(putRecordRequest);
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getRecords() == null) ? 0 : getRecords().hashCode()); hashCode = prime * hashCode + ((getStreamName() == null) ? 0 : getStreamName().hashCode()); return hashCode; }
protected void flush() { kinesis.putRecords(new PutRecordsRequest() .withStreamName(STREAM_NAME) .withRecords(entries)); entries.clear(); }
message = new GenericMessage<>(new PutRecordsRequest() .withStreamName("myStream") .withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); assertThat(putRecordsRequest.getStreamName()).isEqualTo("myStream"); assertThat(putRecordsRequest.getRecords()) .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes()))
@Override public List<byte[]> sendBatch(String streamName, List<byte[]> data) { if (data == null || data.isEmpty()) { return Collections.emptyList(); } final PutRecordsRequest request = new PutRecordsRequest(); request.setStreamName(streamName); final List<PutRecordsRequestEntry> records = new ArrayList<PutRecordsRequestEntry>(data.size()); for (final byte[] d : data) { final String partKey = StringUtils.isBlank(this.partitionKey) ? UUID.randomUUID().toString() : this.partitionKey; final PutRecordsRequestEntry r = new PutRecordsRequestEntry(); r.setData(ByteBuffer.wrap(d)); r.setPartitionKey(partKey); records.add(r); } request.setRecords(records); request.getRequestClientOptions().appendUserAgent(userAgent); final PutRecordsResult result = client.putRecords(request); final int size = result.getRecords().size(); final List<byte[]> failures = new ArrayList<byte[]>(result.getFailedRecordCount()); for (int i = 0; i < size; i++) { if (result.getRecords().get(i).getErrorCode() != null) { // always retry failed record failures.add(data.get(i)); } } return failures; }
public Optional<PutRecordsRequest> flush() { if (entries.size() > 0) { PutRecordsRequest r = new PutRecordsRequest(); r.setRecords(entries); entries = new ArrayList<>(); requestSize = 0; return Optional.of(r); } else { return Optional.empty(); } } }
/** * <p> * The records associated with the request. * </p> * * @param records * The records associated with the request. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequest withRecords(java.util.Collection<PutRecordsRequestEntry> records) { setRecords(records); return this; }
.willAnswer(invocation -> { AsyncHandler<PutRecordsRequest, PutRecordsResult> handler = invocation.getArgument(1); handler.onSuccess(new PutRecordsRequest(), new PutRecordsResult()); return mock(Future.class); })
long totalSize = putRecordsRequest.getRecords().stream().mapToLong(record -> record.getData().limit()).sum(); LOGGER.info("Sending {} records ({} bytes)", putRecordsRequest.getRecords().size(), totalSize); for (int i = 0; i < putRecordsResult.getRecords().size(); i++) { PutRecordsResultEntry resultEntry = putRecordsResult.getRecords().get(i); PutRecordsRequestEntry requestEntry = putRecordsRequest.getRecords().get(i); if (resultEntry.getErrorCode() != null) { entriesForRetry.add(requestEntry); putRecordsRequest.withRecords(entriesForRetry); if (entriesForRetry.size() > 0) { LOGGER.warn("Retrying {} records", entriesForRetry.size());
/** * <p> * The stream name associated with the request. * </p> * * @param streamName * The stream name associated with the request. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequest withStreamName(String streamName) { setStreamName(streamName); return this; }
Mockito.verify(client).putRecords(argument.capture()); String userAgent = argument.getValue().getRequestClientOptions() .getClientMarker(Marker.USER_AGENT); final List<PutRecordsRequestEntry> records = argument.getValue().getRecords(); HashSet partitionKeys = new HashSet(); for(PutRecordsRequestEntry record:records)
/** * <p> * The records associated with the request. * </p> * <p> * Returns a reference to this object so that method calls can be chained * together. * * @param records <p> * The records associated with the request. * </p> * @return A reference to this updated object so that method calls can be * chained together. */ public PutRecordsRequest withRecords(PutRecordsRequestEntry... records) { if (getRecords() == null) { this.records = new java.util.ArrayList<PutRecordsRequestEntry>(records.length); } for (PutRecordsRequestEntry value : records) { this.records.add(value); } return this; }
@Override protected void flush() { PutRecordsRequest req = new PutRecordsRequest() .withStreamName(STREAM_NAME) .withRecords(entries);
.getData()).isEqualTo(ByteBuffer.wrap("message".getBytes())); message = new GenericMessage<>(new PutRecordsRequest() .withStreamName("myStream") .withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); assertThat(((PutRecordsRequest) success.getPayload()).getRecords()) .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey")); message = new GenericMessage<>(new PutRecordsRequest() .withStreamName("myStream") .withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); AwsRequestFailureException putRecordsFailure = (AwsRequestFailureException) failed.getPayload(); assertThat(putRecordsFailure.getCause().getMessage()).isEqualTo("putRecordsRequestEx"); assertThat(((PutRecordsRequest) putRecordsFailure.getRequest()).getStreamName()).isEqualTo("myStream"); assertThat(((PutRecordsRequest) putRecordsFailure.getRequest()).getRecords()) .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes()))
/** * <p> * The records associated with the request. * </p> * <p> * <b>NOTE:</b> This method appends the values to the existing list (if any). Use * {@link #setRecords(java.util.Collection)} or {@link #withRecords(java.util.Collection)} if you want to override * the existing values. * </p> * * @param records * The records associated with the request. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequest withRecords(PutRecordsRequestEntry... records) { if (this.records == null) { setRecords(new com.amazonaws.internal.SdkInternalList<PutRecordsRequestEntry>(records.length)); } for (PutRecordsRequestEntry ele : records) { this.records.add(ele); } return this; }
/** * <p> * The stream name associated with the request. * </p> * * @param streamName * The stream name associated with the request. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequest withStreamName(String streamName) { setStreamName(streamName); return this; }
@Test public void test() throws Exception { PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); putRecordsRequest.setStreamName("stream name"); List<PutRecordsRequestEntry> records = new ArrayList<PutRecordsRequestEntry>(); for (int i = 0; i < 10; i++) { PutRecordsRequestEntry record = new PutRecordsRequestEntry(); String randomStr = RandomStringUtils.random(8 * 1024); record.setData(ByteBuffer.wrap(randomStr.getBytes(StringUtils.UTF8))); record.setPartitionKey("partition key"); records.add(record); } putRecordsRequest.setRecords(records); PutRecordsRequestMarshaller marshaller = new PutRecordsRequestMarshaller(); Request<PutRecordsRequest> request = marshaller.marshall(putRecordsRequest); assertEquals("content encoding", "gzip", request.getHeaders().get("Content-Encoding")); byte[] content = IOUtils.toByteArray(request.getContent()); assertEquals("content length", request.getHeaders().get("Content-Length"), String.valueOf(content.length)); GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(content)); String str = IOUtils.toString(gis); assertTrue("content is compressed", content.length < str.length()); Map<String, String> map = JsonUtils.jsonToMap(str); assertEquals("StreamName", "stream name", map.get("StreamName")); } }
public Observable<PutRecordsResultEntry> play(LocalDateTime start, @Nullable LocalDateTime end) { return playableObjects(start, end) .onBackpressureBuffer() .observeOn(Schedulers.io()) .flatMap(this::objectToPayloads) .map(ByteBuffer::wrap) .lift(new OperatorBufferKinesisBatch(MAX_KINESIS_BATCH_SIZE, MAX_KINESIS_BATCH_WEIGHT)) .onBackpressureBuffer() .map(byteBuffers -> byteBuffers.stream() .map(buffer -> new PutRecordsRequestEntry() .withData(buffer) .withPartitionKey(UUID.randomUUID().toString())) .collect(toList())) .map(entries -> new PutRecordsRequest() .withStreamName(vcrConfiguration.targetStream) .withRecords(entries)) .observeOn(Schedulers.io()) .flatMap(putRecordsRequest -> Observable.create((Observable.OnSubscribe<List<PutRecordsResult>>) os -> { os.onStart(); kinesisWriter.submit(() -> { os.onNext(putWithRetry(putRecordsRequest).orElse(Collections.<PutRecordsResult>emptyList())); os.onCompleted(); }); })) .flatMap(Observable::from) .flatMap(putRecordsResult -> Observable.from(putRecordsResult.getRecords())) .doOnNext(result -> LOGGER.debug("Wrote record. Seq {}, shard {}", result.getSequenceNumber(), result.getShardId())); }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof PutRecordsRequest == false) return false; PutRecordsRequest other = (PutRecordsRequest) obj; if (other.getRecords() == null ^ this.getRecords() == null) return false; if (other.getRecords() != null && other.getRecords().equals(this.getRecords()) == false) return false; if (other.getStreamName() == null ^ this.getStreamName() == null) return false; if (other.getStreamName() != null && other.getStreamName().equals(this.getStreamName()) == false) return false; return true; }
/** * <p> * The records associated with the request. * </p> * <p> * Returns a reference to this object so that method calls can be chained * together. * * @param records <p> * The records associated with the request. * </p> * @return A reference to this updated object so that method calls can be * chained together. */ public PutRecordsRequest withRecords(java.util.Collection<PutRecordsRequestEntry> records) { setRecords(records); return this; }