@Test(timeout=30000) public void testCopyFromHostConnectionTimeout() throws Exception { when(connection.getInputStream()).thenThrow( new SocketTimeoutException("This is a fake timeout :)")); Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm, r, metrics, except, key, connection); underTest.copyFromHost(host); verify(connection).addRequestProperty( SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash); verify(allErrs).increment(1); verify(ss).copyFailed(map1ID, host, false, false); verify(ss).copyFailed(map2ID, host, false, false); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); }
@Test public void testCopyFromHostIncompatibleShuffleVersionWithRetry() throws Exception { String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); when(connection.getResponseCode()).thenReturn(200); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME)) .thenReturn("mapreduce").thenReturn("other").thenReturn("other"); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION)) .thenReturn("1.0.1").thenReturn("1.0.0").thenReturn("1.0.1"); when(connection.getHeaderField( SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)).thenReturn(replyHash); ByteArrayInputStream in = new ByteArrayInputStream(new byte[0]); when(connection.getInputStream()).thenReturn(in); for (int i = 0; i < 3; ++i) { Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(jobWithRetry, id, ss, mm, r, metrics, except, key, connection); underTest.copyFromHost(host); } verify(connection, times(3)).addRequestProperty( SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash); verify(allErrs, times(3)).increment(1); verify(ss, times(3)).copyFailed(map1ID, host, false, false); verify(ss, times(3)).copyFailed(map2ID, host, false, false); verify(ss, times(3)).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss, times(3)).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); }
@Test public void testCopyFromHostBogusHeader() throws Exception { Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm, r, metrics, except, key, connection); String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); when(connection.getResponseCode()).thenReturn(200); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME)) .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION)) .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)) .thenReturn(replyHash); ByteArrayInputStream in = new ByteArrayInputStream( "\u00010 BOGUS DATA\nBOGUS DATA\nBOGUS DATA\n".getBytes()); when(connection.getInputStream()).thenReturn(in); underTest.copyFromHost(host); verify(connection).addRequestProperty( SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash); verify(allErrs).increment(1); verify(ss).copyFailed(map1ID, host, true, false); verify(ss).copyFailed(map2ID, host, true, false); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); }
@Test public void testCopyFromHostIncompatibleShuffleVersion() throws Exception { String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); when(connection.getResponseCode()).thenReturn(200); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME)) .thenReturn("mapreduce").thenReturn("other").thenReturn("other"); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION)) .thenReturn("1.0.1").thenReturn("1.0.0").thenReturn("1.0.1"); when(connection.getHeaderField( SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)).thenReturn(replyHash); ByteArrayInputStream in = new ByteArrayInputStream(new byte[0]); when(connection.getInputStream()).thenReturn(in); for (int i = 0; i < 3; ++i) { Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm, r, metrics, except, key, connection); underTest.copyFromHost(host); } verify(connection, times(3)).addRequestProperty( SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash); verify(allErrs, times(3)).increment(1); verify(ss, times(3)).copyFailed(map1ID, host, false, false); verify(ss, times(3)).copyFailed(map2ID, host, false, false); verify(ss, times(3)).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss, times(3)).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); }
verify(ss, never()).copyFailed(map2ID, host, true, false); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID));
verify(ss, times(1)).copyFailed(map1ID, host, false, false); verify(ss, times(1)).copyFailed(map2ID, host, false, false); verify(ss, times(1)).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss, times(1)).putBackKnownMapOutput(any(MapHost.class), eq(map2ID));
@Test public void testCopyFromHostConnectionRejected() throws Exception { when(connection.getResponseCode()) .thenReturn(Fetcher.TOO_MANY_REQ_STATUS_CODE); Fetcher<Text, Text> fetcher = new FakeFetcher<>(job, id, ss, mm, r, metrics, except, key, connection); fetcher.copyFromHost(host); Assert.assertEquals("No host failure is expected.", ss.hostFailureCount(host.getHostName()), 0); Assert.assertEquals("No fetch failure is expected.", ss.fetchFailureCount(map1ID), 0); Assert.assertEquals("No fetch failure is expected.", ss.fetchFailureCount(map2ID), 0); verify(ss).penalize(eq(host), anyLong()); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); }
verify(ss, never()).copyFailed(map2ID, host, true, false); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID));
private void abortConnect(MapHost host, Set<TaskAttemptID> remaining) { for (TaskAttemptID left : remaining) { scheduler.putBackKnownMapOutput(host, left); } closeConnection(); }
private void abortConnect(MapHost host, Set<TaskAttemptID> remaining) { for (TaskAttemptID left : remaining) { scheduler.putBackKnownMapOutput(host, left); } closeConnection(); }
private void abortConnect(MapHost host, Set<TaskAttemptID> remaining) { for (TaskAttemptID left : remaining) { scheduler.putBackKnownMapOutput(host, left); } closeConnection(); }
private void abortConnect(MapHost host, Set<TaskAttemptID> remaining) { for (TaskAttemptID left : remaining) { scheduler.putBackKnownMapOutput(host, left); } closeConnection(); }
private DataInputStream openShuffleUrl(MapHost host, Set<TaskAttemptID> remaining, URL url) { DataInputStream input = null; try { setupConnectionsWithRetry(url); if (stopped) { abortConnect(host, remaining); } else { input = new DataInputStream(connection.getInputStream()); } } catch (IOException ie) { boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } // Add back all the remaining maps, WITHOUT marking them as failed for(TaskAttemptID left: remaining) { scheduler.putBackKnownMapOutput(host, left); } } return input; }
private DataInputStream openShuffleUrl(MapHost host, Set<TaskAttemptID> remaining, URL url) { DataInputStream input = null; try { setupConnectionsWithRetry(host, remaining, url); if (stopped) { abortConnect(host, remaining); } else { input = new DataInputStream(connection.getInputStream()); } } catch (IOException ie) { boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } // Add back all the remaining maps, WITHOUT marking them as failed for(TaskAttemptID left: remaining) { scheduler.putBackKnownMapOutput(host, left); } } return input; }
scheduler.putBackKnownMapOutput(host, left);
scheduler.putBackKnownMapOutput(host, left);
scheduler.putBackKnownMapOutput(host, left);
private DataInputStream openShuffleUrl(MapHost host, Set<TaskAttemptID> remaining, URL url) { DataInputStream input = null; try { setupConnectionsWithRetry(host, remaining, url); if (stopped) { abortConnect(host, remaining); } else { input = new DataInputStream(connection.getInputStream()); } } catch (IOException ie) { boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } // Add back all the remaining maps, WITHOUT marking them as failed for(TaskAttemptID left: remaining) { scheduler.putBackKnownMapOutput(host, left); } } return input; }
scheduler.putBackKnownMapOutput(host, left);