/** Invokes the "onBlockFetchFailure" callback for every listed block id. */ private void failRemainingBlocks(String[] failedBlockIds, Throwable e) { for (String blockId : failedBlockIds) { try { listener.onBlockFetchFailure(blockId, e); } catch (Exception e2) { logger.error("Error in block fetch failure callback", e2); } } }
/** Invokes the "onBlockFetchFailure" callback for every listed block id. */ private void failRemainingBlocks(String[] failedBlockIds, Throwable e) { for (String blockId : failedBlockIds) { try { listener.onBlockFetchFailure(blockId, e); } catch (Exception e2) { logger.error("Error in block fetch failure callback", e2); } } }
/** Invokes the "onBlockFetchFailure" callback for every listed block id. */ private void failRemainingBlocks(String[] failedBlockIds, Throwable e) { for (String blockId : failedBlockIds) { try { listener.onBlockFetchFailure(blockId, e); } catch (Exception e2) { logger.error("Error in block fetch failure callback", e2); } } }
@Override public void onBlockFetchFailure(String blockId, Throwable exception) { // We will only forward this failure to our parent listener if this block request is // outstanding, we are still the active listener, AND we cannot retry the fetch. boolean shouldForwardFailure = false; synchronized (RetryingBlockFetcher.this) { if (this == currentListener && outstandingBlocksIds.contains(blockId)) { if (shouldRetry(exception)) { initiateRetry(); } else { logger.error(String.format("Failed to fetch block %s, and will not retry (%s retries)", blockId, retryCount), exception); outstandingBlocksIds.remove(blockId); shouldForwardFailure = true; } } } // Now actually invoke the parent listener, outside of the synchronized block. if (shouldForwardFailure) { listener.onBlockFetchFailure(blockId, exception); } } }
@Override public void onBlockFetchFailure(String blockId, Throwable exception) { // We will only forward this failure to our parent listener if this block request is // outstanding, we are still the active listener, AND we cannot retry the fetch. boolean shouldForwardFailure = false; synchronized (RetryingBlockFetcher.this) { if (this == currentListener && outstandingBlocksIds.contains(blockId)) { if (shouldRetry(exception)) { initiateRetry(); } else { logger.error(String.format("Failed to fetch block %s, and will not retry (%s retries)", blockId, retryCount), exception); outstandingBlocksIds.remove(blockId); shouldForwardFailure = true; } } } // Now actually invoke the parent listener, outside of the synchronized block. if (shouldForwardFailure) { listener.onBlockFetchFailure(blockId, exception); } } }
@Override public void onBlockFetchFailure(String blockId, Throwable exception) { // We will only forward this failure to our parent listener if this block request is // outstanding, we are still the active listener, AND we cannot retry the fetch. boolean shouldForwardFailure = false; synchronized (RetryingBlockFetcher.this) { if (this == currentListener && outstandingBlocksIds.contains(blockId)) { if (shouldRetry(exception)) { initiateRetry(); } else { logger.error(String.format("Failed to fetch block %s, and will not retry (%s retries)", blockId, retryCount), exception); outstandingBlocksIds.remove(blockId); shouldForwardFailure = true; } } } // Now actually invoke the parent listener, outside of the synchronized block. if (shouldForwardFailure) { listener.onBlockFetchFailure(blockId, exception); } } }
/** * Fires off a request to fetch all blocks that have not been fetched successfully or permanently * failed (i.e., by a non-IOException). */ private void fetchAllOutstanding() { // Start by retrieving our shared state within a synchronized block. String[] blockIdsToFetch; int numRetries; RetryingBlockFetchListener myListener; synchronized (this) { blockIdsToFetch = outstandingBlocksIds.toArray(new String[outstandingBlocksIds.size()]); numRetries = retryCount; myListener = currentListener; } // Now initiate the fetch on all outstanding blocks, possibly initiating a retry if that fails. try { fetchStarter.createAndStart(blockIdsToFetch, myListener); } catch (Exception e) { logger.error(String.format("Exception while beginning fetch of %s outstanding blocks %s", blockIdsToFetch.length, numRetries > 0 ? "(after " + numRetries + " retries)" : ""), e); if (shouldRetry(e)) { initiateRetry(); } else { for (String bid : blockIdsToFetch) { listener.onBlockFetchFailure(bid, e); } } } }
@Test public void testFailure() { LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap(); blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); blocks.put("b1", null); blocks.put("b2", null); BlockFetchingListener listener = fetchBlocks(blocks); // Each failure will cause a failure to be invoked in all remaining block fetches. verify(listener, times(1)).onBlockFetchSuccess("b0", blocks.get("b0")); verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); verify(listener, times(2)).onBlockFetchFailure(eq("b2"), any()); }
@Test public void testFailure() { LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap(); blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); blocks.put("b1", null); blocks.put("b2", null); BlockFetchingListener listener = fetchBlocks(blocks); // Each failure will cause a failure to be invoked in all remaining block fetches. verify(listener, times(1)).onBlockFetchSuccess("b0", blocks.get("b0")); verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); verify(listener, times(2)).onBlockFetchFailure(eq("b2"), any()); }
@Test public void testFailure() { LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap(); blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); blocks.put("b1", null); blocks.put("b2", null); BlockFetchingListener listener = fetchBlocks(blocks); // Each failure will cause a failure to be invoked in all remaining block fetches. verify(listener, times(1)).onBlockFetchSuccess("b0", blocks.get("b0")); verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); verify(listener, times(2)).onBlockFetchFailure(eq("b2"), any()); }
@Test public void testFailureAndSuccess() { LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap(); blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); blocks.put("b1", null); blocks.put("b2", new NioManagedBuffer(ByteBuffer.wrap(new byte[21]))); BlockFetchingListener listener = fetchBlocks(blocks); // We may call both success and failure for the same block. verify(listener, times(1)).onBlockFetchSuccess("b0", blocks.get("b0")); verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); verify(listener, times(1)).onBlockFetchSuccess("b2", blocks.get("b2")); verify(listener, times(1)).onBlockFetchFailure(eq("b2"), any()); }
@Test public void testFailureAndSuccess() { LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap(); blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); blocks.put("b1", null); blocks.put("b2", new NioManagedBuffer(ByteBuffer.wrap(new byte[21]))); BlockFetchingListener listener = fetchBlocks(blocks); // We may call both success and failure for the same block. verify(listener, times(1)).onBlockFetchSuccess("b0", blocks.get("b0")); verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); verify(listener, times(1)).onBlockFetchSuccess("b2", blocks.get("b2")); verify(listener, times(1)).onBlockFetchFailure(eq("b2"), any()); }
@Test public void testUnrecoverableFailure() throws IOException, InterruptedException { BlockFetchingListener listener = mock(BlockFetchingListener.class); List<? extends Map<String, Object>> interactions = Arrays.asList( // b0 throws a non-IOException error, so it will be failed without retry. ImmutableMap.<String, Object>builder() .put("b0", new RuntimeException("Ouch!")) .put("b1", block1) .build() ); performInteractions(interactions, listener); verify(listener).onBlockFetchFailure(eq("b0"), any()); verify(listener).onBlockFetchSuccess("b1", block1); verifyNoMoreInteractions(listener); }
@Test public void testFailureAndSuccess() { LinkedHashMap<String, ManagedBuffer> blocks = Maps.newLinkedHashMap(); blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); blocks.put("b1", null); blocks.put("b2", new NioManagedBuffer(ByteBuffer.wrap(new byte[21]))); BlockFetchingListener listener = fetchBlocks(blocks); // We may call both success and failure for the same block. verify(listener, times(1)).onBlockFetchSuccess("b0", blocks.get("b0")); verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); verify(listener, times(1)).onBlockFetchSuccess("b2", blocks.get("b2")); verify(listener, times(1)).onBlockFetchFailure(eq("b2"), any()); }
@Test public void testUnrecoverableFailure() throws IOException, InterruptedException { BlockFetchingListener listener = mock(BlockFetchingListener.class); List<? extends Map<String, Object>> interactions = Arrays.asList( // b0 throws a non-IOException error, so it will be failed without retry. ImmutableMap.<String, Object>builder() .put("b0", new RuntimeException("Ouch!")) .put("b1", block1) .build() ); performInteractions(interactions, listener); verify(listener).onBlockFetchFailure(eq("b0"), any()); verify(listener).onBlockFetchSuccess("b1", block1); verifyNoMoreInteractions(listener); }
@Test public void testUnrecoverableFailure() throws IOException, InterruptedException { BlockFetchingListener listener = mock(BlockFetchingListener.class); List<? extends Map<String, Object>> interactions = Arrays.asList( // b0 throws a non-IOException error, so it will be failed without retry. ImmutableMap.<String, Object>builder() .put("b0", new RuntimeException("Ouch!")) .put("b1", block1) .build() ); performInteractions(interactions, listener); verify(listener).onBlockFetchFailure(eq("b0"), any()); verify(listener).onBlockFetchSuccess("b1", block1); verifyNoMoreInteractions(listener); }
@Test public void testThreeIOExceptions() throws IOException, InterruptedException { BlockFetchingListener listener = mock(BlockFetchingListener.class); List<? extends Map<String, Object>> interactions = Arrays.asList( // b0's IOException will trigger retry, b1's will be ignored. ImmutableMap.<String, Object>builder() .put("b0", new IOException()) .put("b1", new IOException()) .build(), // Next, b0 is successful and b1 errors again, so we just request that one. ImmutableMap.<String, Object>builder() .put("b0", block0) .put("b1", new IOException()) .build(), // b1 errors again, but this was the last retry ImmutableMap.<String, Object>builder() .put("b1", new IOException()) .build(), // This is not reached -- b1 has failed. ImmutableMap.<String, Object>builder() .put("b1", block1) .build() ); performInteractions(interactions, listener); verify(listener, timeout(5000)).onBlockFetchSuccess("b0", block0); verify(listener, timeout(5000)).onBlockFetchFailure(eq("b1"), any()); verifyNoMoreInteractions(listener); }
@Test public void testRetryAndUnrecoverable() throws IOException, InterruptedException { BlockFetchingListener listener = mock(BlockFetchingListener.class); List<? extends Map<String, Object>> interactions = Arrays.asList( // b0's IOException will trigger retry, subsequent messages will be ignored. ImmutableMap.<String, Object>builder() .put("b0", new IOException()) .put("b1", new RuntimeException()) .put("b2", block2) .build(), // Next, b0 is successful, b1 errors unrecoverably, and b2 triggers a retry. ImmutableMap.<String, Object>builder() .put("b0", block0) .put("b1", new RuntimeException()) .put("b2", new IOException()) .build(), // b2 succeeds in its last retry. ImmutableMap.<String, Object>builder() .put("b2", block2) .build() ); performInteractions(interactions, listener); verify(listener, timeout(5000)).onBlockFetchSuccess("b0", block0); verify(listener, timeout(5000)).onBlockFetchFailure(eq("b1"), any()); verify(listener, timeout(5000)).onBlockFetchSuccess("b2", block2); verifyNoMoreInteractions(listener); }
@Test public void testRetryAndUnrecoverable() throws IOException, InterruptedException { BlockFetchingListener listener = mock(BlockFetchingListener.class); List<? extends Map<String, Object>> interactions = Arrays.asList( // b0's IOException will trigger retry, subsequent messages will be ignored. ImmutableMap.<String, Object>builder() .put("b0", new IOException()) .put("b1", new RuntimeException()) .put("b2", block2) .build(), // Next, b0 is successful, b1 errors unrecoverably, and b2 triggers a retry. ImmutableMap.<String, Object>builder() .put("b0", block0) .put("b1", new RuntimeException()) .put("b2", new IOException()) .build(), // b2 succeeds in its last retry. ImmutableMap.<String, Object>builder() .put("b2", block2) .build() ); performInteractions(interactions, listener); verify(listener, timeout(5000)).onBlockFetchSuccess("b0", block0); verify(listener, timeout(5000)).onBlockFetchFailure(eq("b1"), any()); verify(listener, timeout(5000)).onBlockFetchSuccess("b2", block2); verifyNoMoreInteractions(listener); }
@Test public void testRetryAndUnrecoverable() throws IOException, InterruptedException { BlockFetchingListener listener = mock(BlockFetchingListener.class); List<? extends Map<String, Object>> interactions = Arrays.asList( // b0's IOException will trigger retry, subsequent messages will be ignored. ImmutableMap.<String, Object>builder() .put("b0", new IOException()) .put("b1", new RuntimeException()) .put("b2", block2) .build(), // Next, b0 is successful, b1 errors unrecoverably, and b2 triggers a retry. ImmutableMap.<String, Object>builder() .put("b0", block0) .put("b1", new RuntimeException()) .put("b2", new IOException()) .build(), // b2 succeeds in its last retry. ImmutableMap.<String, Object>builder() .put("b2", block2) .build() ); performInteractions(interactions, listener); verify(listener, timeout(5000)).onBlockFetchSuccess("b0", block0); verify(listener, timeout(5000)).onBlockFetchFailure(eq("b1"), any()); verify(listener, timeout(5000)).onBlockFetchSuccess("b2", block2); verifyNoMoreInteractions(listener); }