@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getUnprocessedItems() == null) ? 0 : getUnprocessedItems().hashCode()); hashCode = prime * hashCode + ((getItemCollectionMetrics() == null) ? 0 : getItemCollectionMetrics().hashCode()); hashCode = prime * hashCode + ((getConsumedCapacity() == null) ? 0 : getConsumedCapacity().hashCode()); return hashCode; }
public BatchWriteItemResult unmarshall(JsonUnmarshallerContext context) throws Exception { BatchWriteItemResult batchWriteItemResult = new BatchWriteItemResult(); if (context.testExpression("UnprocessedItems", targetDepth)) { context.nextToken(); batchWriteItemResult.setUnprocessedItems(new MapUnmarshaller<String, java.util.List<WriteRequest>>(context.getUnmarshaller(String.class), new ListUnmarshaller<WriteRequest>(WriteRequestJsonUnmarshaller.getInstance())).unmarshall(context)); batchWriteItemResult.setItemCollectionMetrics(new MapUnmarshaller<String, java.util.List<ItemCollectionMetrics>>(context .getUnmarshaller(String.class), new ListUnmarshaller<ItemCollectionMetrics>(ItemCollectionMetricsJsonUnmarshaller.getInstance())) .unmarshall(context)); batchWriteItemResult.setConsumedCapacity(new ListUnmarshaller<ConsumedCapacity>(ConsumedCapacityJsonUnmarshaller.getInstance()) .unmarshall(context));
private void batchWrite(Map<String, List<WriteRequest>> items) { if (items == null || items.isEmpty()) { return; } try { BatchWriteItemResult result = client().batchWriteItem(new BatchWriteItemRequest(). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withRequestItems(items)); if (result == null) { return; } logger.debug("batchWrite() CC: {}", result.getConsumedCapacity()); if (result.getUnprocessedItems() != null && !result.getUnprocessedItems().isEmpty()) { Thread.sleep(1000); logger.warn("UNPROCESSED {0}", result.getUnprocessedItems().size()); batchWrite(result.getUnprocessedItems()); } } catch (Exception e) { logger.error(null, e); } }
/** * Convenient method to return the low-level unprocessed items. * * @see BatchWriteItemApi#batchWriteItemUnprocessed(Map) * @see BatchWriteItemSpec#withUnprocessedItems(Map) */ public Map<String, List<WriteRequest>> getUnprocessedItems() { return result.getUnprocessedItems(); }
if (getConsumedCapacity() == null) { this.consumedCapacity = new java.util.ArrayList<ConsumedCapacity>( consumedCapacity.length);
setConsumedCapacity(consumedCapacity); return this;
/** * Writes multiple items in batch. * @param items a map of tables->write requests * @param backoff backoff seconds */ protected static void batchWrite(Map<String, List<WriteRequest>> items, int backoff) { if (items == null || items.isEmpty()) { return; } try { BatchWriteItemResult result = getClient().batchWriteItem(new BatchWriteItemRequest(). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withRequestItems(items)); if (result == null) { return; } logger.debug("batchWrite(): total {}, cc {}", items.size(), result.getConsumedCapacity()); if (result.getUnprocessedItems() != null && !result.getUnprocessedItems().isEmpty()) { Thread.sleep((long) backoff * 1000L); logger.warn("{} UNPROCESSED write requests!", result.getUnprocessedItems().size()); batchWrite(result.getUnprocessedItems(), backoff * 2); } } catch (Exception e) { logger.error(null, e); throwIfNecessary(e); } }
/** * Helper method to handle unprocessed items items * @param session process session * @param keysToFlowFileMap map of flow db primary key to flow file * @param table dynamodb table * @param hashKeyName the hash key name * @param hashKeyValueType the hash key value * @param rangeKeyName the range key name * @param rangeKeyValueType range key value * @param outcome the write outcome */ protected void handleUnprocessedItems(final ProcessSession session, Map<ItemKeys, FlowFile> keysToFlowFileMap, final String table, final String hashKeyName, final String hashKeyValueType, final String rangeKeyName, final String rangeKeyValueType, BatchWriteItemOutcome outcome) { BatchWriteItemResult result = outcome.getBatchWriteItemResult(); // Handle unprocessed items List<WriteRequest> unprocessedItems = result.getUnprocessedItems().get(table); if ( unprocessedItems != null && unprocessedItems.size() > 0 ) { for ( WriteRequest request : unprocessedItems) { Map<String,AttributeValue> item = getRequestItem(request); Object hashKeyValue = getValue(item, hashKeyName, hashKeyValueType); Object rangeKeyValue = getValue(item, rangeKeyName, rangeKeyValueType); sendUnprocessedToUnprocessedRelationship(session, keysToFlowFileMap, hashKeyValue, rangeKeyValue); } } }
@Override public BatchWriteItemResult measureConsumedCapacity(BatchWriteItemResult result) { consumed.addAndGet(result.getConsumedCapacity() == null ? 0 : getConsumedCapacityForTable(result.getConsumedCapacity())); return result; } }
setConsumedCapacity(new java.util.ArrayList<ConsumedCapacity>(consumedCapacity.length));
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof BatchWriteItemResult == false) return false; BatchWriteItemResult other = (BatchWriteItemResult) obj; if (other.getUnprocessedItems() == null ^ this.getUnprocessedItems() == null) return false; if (other.getUnprocessedItems() != null && other.getUnprocessedItems().equals(this.getUnprocessedItems()) == false) return false; if (other.getItemCollectionMetrics() == null ^ this.getItemCollectionMetrics() == null) return false; if (other.getItemCollectionMetrics() != null && other.getItemCollectionMetrics().equals(this.getItemCollectionMetrics()) == false) return false; if (other.getConsumedCapacity() == null ^ this.getConsumedCapacity() == null) return false; if (other.getConsumedCapacity() != null && other.getConsumedCapacity().equals(this.getConsumedCapacity()) == false) return false; return true; }
/** * Writes multiple items in batch. * @param items a map of tables->write requests * @param backoff backoff seconds */ protected static void batchWrite(Map<String, List<WriteRequest>> items, int backoff) { if (items == null || items.isEmpty()) { return; } try { BatchWriteItemResult result = getClient().batchWriteItem(new BatchWriteItemRequest(). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withRequestItems(items)); if (result == null) { return; } logger.debug("batchWrite(): total {}, cc {}", items.size(), result.getConsumedCapacity()); if (result.getUnprocessedItems() != null && !result.getUnprocessedItems().isEmpty()) { Thread.sleep((long) backoff * 1000L); logger.warn("{} UNPROCESSED write requests!", result.getUnprocessedItems().size()); batchWrite(result.getUnprocessedItems(), backoff * 2); } } catch (Exception e) { logger.error(null, e); throwIfNecessary(e); } }
return failedBatch; pendingItems = result.getUnprocessedItems();
apiTimerContext.stop(); if (result.getConsumedCapacity() != null) { for (ConsumedCapacity ccu : result.getConsumedCapacity()) { meterConsumedCapacity(BATCH_WRITE_ITEM, ccu);
public BatchWriteItemResult unmarshall(JsonUnmarshallerContext context) throws Exception { BatchWriteItemResult batchWriteItemResult = new BatchWriteItemResult(); if (name.equals("UnprocessedItems")) { batchWriteItemResult .setUnprocessedItems(new MapUnmarshaller<java.util.List<WriteRequest>>( new ListUnmarshaller<WriteRequest>(WriteRequestJsonUnmarshaller .getInstance() } else if (name.equals("ItemCollectionMetrics")) { batchWriteItemResult .setItemCollectionMetrics(new MapUnmarshaller<java.util.List<ItemCollectionMetrics>>( new ListUnmarshaller<ItemCollectionMetrics>( ItemCollectionMetricsJsonUnmarshaller.getInstance() batchWriteItemResult.setConsumedCapacity(new ListUnmarshaller<ConsumedCapacity>( ConsumedCapacityJsonUnmarshaller.getInstance()
setConsumedCapacity(consumedCapacity); return this;
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getUnprocessedItems() != null) sb.append("UnprocessedItems: ").append(getUnprocessedItems()).append(","); if (getItemCollectionMetrics() != null) sb.append("ItemCollectionMetrics: ").append(getItemCollectionMetrics()).append(","); if (getConsumedCapacity() != null) sb.append("ConsumedCapacity: ").append(getConsumedCapacity()); sb.append("}"); return sb.toString(); }
do { writeItemResult = client.batchWriteItem(req); unprocessedItems = writeItemResult.getUnprocessedItems(); consumedCapacities .addAll(writeItemResult.getConsumedCapacity());
/** * Continue trying to process the batch until it finishes or an exception * occurs. */ private FailedBatch callUntilCompletion(Map<String, List<WriteRequest>> batch) { BatchWriteItemResult result = null; int retries = 0; FailedBatch failedBatch = null; while (true) { try { result = db.batchWriteItem(applyBatchOperationUserAgent( new BatchWriteItemRequest().withRequestItems(batch))); } catch (final Exception e) { failedBatch = new FailedBatch(); failedBatch.setUnprocessedItems(batch); failedBatch.setException(e); return failedBatch; } retries++; batch = result.getUnprocessedItems(); if (batch.size() > 0) { pauseExponentially(retries); } else { break; } } return failedBatch; }
public BatchWriteItemResult unmarshall(JsonUnmarshallerContext context) throws Exception { BatchWriteItemResult batchWriteItemResult = new BatchWriteItemResult(); if (context.testExpression("UnprocessedItems", targetDepth)) { context.nextToken(); batchWriteItemResult.setUnprocessedItems(new MapUnmarshaller<String, java.util.List<WriteRequest>>(context.getUnmarshaller(String.class), new ListUnmarshaller<WriteRequest>(WriteRequestJsonUnmarshaller.getInstance())).unmarshall(context)); batchWriteItemResult.setItemCollectionMetrics(new MapUnmarshaller<String, java.util.List<ItemCollectionMetrics>>(context .getUnmarshaller(String.class), new ListUnmarshaller<ItemCollectionMetrics>(ItemCollectionMetricsJsonUnmarshaller.getInstance())) .unmarshall(context)); batchWriteItemResult.setConsumedCapacity(new ListUnmarshaller<ConsumedCapacity>(ConsumedCapacityJsonUnmarshaller.getInstance()) .unmarshall(context));