@BeforeClass public static void setUp() { conf = new TransportConf("rpc", MapConfigProvider.EMPTY); }
@BeforeClass public static void setUp() { conf = new TransportConf("rpc", MapConfigProvider.EMPTY); }
@Before public void setUp() throws Exception { Map<String, String> configMap = new HashMap<>(); configMap.put("spark.shuffle.io.connectionTimeout", "10s"); conf = new TransportConf("shuffle", new MapConfigProvider(configMap)); defaultManager = new StreamManager() { @Override public ManagedBuffer getChunk(long streamId, int chunkIndex) { throw new UnsupportedOperationException(); } }; }
@Before public void setUp() throws Exception { Map<String, String> configMap = new HashMap<>(); configMap.put("spark.shuffle.io.connectionTimeout", "10s"); conf = new TransportConf("shuffle", new MapConfigProvider(configMap)); defaultManager = new StreamManager() { @Override public ManagedBuffer getChunk(long streamId, int chunkIndex) { throw new UnsupportedOperationException(); } }; }
@BeforeClass public static void beforeAll() throws IOException { conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); context = new TransportContext(conf, new TestRpcHandler()); secretKeyHolder = mock(SecretKeyHolder.class); when(secretKeyHolder.getSaslUser(eq("app-1"))).thenReturn("app-1"); when(secretKeyHolder.getSecretKey(eq("app-1"))).thenReturn("app-1"); when(secretKeyHolder.getSaslUser(eq("app-2"))).thenReturn("app-2"); when(secretKeyHolder.getSecretKey(eq("app-2"))).thenReturn("app-2"); when(secretKeyHolder.getSaslUser(anyString())).thenReturn("other-app"); when(secretKeyHolder.getSecretKey(anyString())).thenReturn("correct-password"); TransportServerBootstrap bootstrap = new SaslServerBootstrap(conf, secretKeyHolder); server = context.createServer(Arrays.asList(bootstrap)); }
@BeforeClass public static void beforeAll() throws IOException { conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); context = new TransportContext(conf, new TestRpcHandler()); secretKeyHolder = mock(SecretKeyHolder.class); when(secretKeyHolder.getSaslUser(eq("app-1"))).thenReturn("app-1"); when(secretKeyHolder.getSecretKey(eq("app-1"))).thenReturn("app-1"); when(secretKeyHolder.getSaslUser(eq("app-2"))).thenReturn("app-2"); when(secretKeyHolder.getSecretKey(eq("app-2"))).thenReturn("app-2"); when(secretKeyHolder.getSaslUser(anyString())).thenReturn("other-app"); when(secretKeyHolder.getSecretKey(anyString())).thenReturn("correct-password"); TransportServerBootstrap bootstrap = new SaslServerBootstrap(conf, secretKeyHolder); server = context.createServer(Arrays.asList(bootstrap)); }
@BeforeClass public static void beforeAll() throws IOException { conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); context = new TransportContext(conf, new TestRpcHandler()); secretKeyHolder = mock(SecretKeyHolder.class); when(secretKeyHolder.getSaslUser(eq("app-1"))).thenReturn("app-1"); when(secretKeyHolder.getSecretKey(eq("app-1"))).thenReturn("app-1"); when(secretKeyHolder.getSaslUser(eq("app-2"))).thenReturn("app-2"); when(secretKeyHolder.getSecretKey(eq("app-2"))).thenReturn("app-2"); when(secretKeyHolder.getSaslUser(anyString())).thenReturn("other-app"); when(secretKeyHolder.getSecretKey(anyString())).thenReturn("correct-password"); TransportServerBootstrap bootstrap = new SaslServerBootstrap(conf, secretKeyHolder); server = context.createServer(Arrays.asList(bootstrap)); }
private void setUp(boolean enableVerboseMetrics) { HashMap<String, String> configMap = new HashMap<>(); configMap.put("spark.shuffle.io.enableVerboseMetrics", String.valueOf(enableVerboseMetrics)); conf = new TransportConf("shuffle", new MapConfigProvider(configMap)); RpcHandler rpcHandler = new NoOpRpcHandler(); context = new TransportContext(conf, rpcHandler); server = context.createServer(); clientFactory = context.createClientFactory(); }
@Test public void testFetchNoServer() throws Exception { TransportConf clientConf = new TransportConf("shuffle", new MapConfigProvider(ImmutableMap.of("spark.shuffle.io.maxRetries", "0"))); registerExecutor("exec-0", dataContext0.createExecutorInfo(SORT_MANAGER)); FetchResult execFetch = fetchBlocks("exec-0", new String[]{"shuffle_1_0_0", "shuffle_1_0_1"}, clientConf, 1 /* port */); assertTrue(execFetch.successBlocks.isEmpty()); assertEquals(Sets.newHashSet("shuffle_1_0_0", "shuffle_1_0_1"), execFetch.failedBlocks); }
@Test public void testFetchNoServer() throws Exception { TransportConf clientConf = new TransportConf("shuffle", new MapConfigProvider(ImmutableMap.of("spark.shuffle.io.maxRetries", "0"))); registerExecutor("exec-0", dataContext0.createExecutorInfo(SORT_MANAGER)); FetchResult execFetch = fetchBlocks("exec-0", new String[]{"shuffle_1_0_0", "shuffle_1_0_1"}, clientConf, 1 /* port */); assertTrue(execFetch.successBlocks.isEmpty()); assertEquals(Sets.newHashSet("shuffle_1_0_0", "shuffle_1_0_1"), execFetch.failedBlocks); }
@Test public void testFetchNoServer() throws Exception { TransportConf clientConf = new TransportConf("shuffle", new MapConfigProvider(ImmutableMap.of("spark.shuffle.io.maxRetries", "0"))); registerExecutor("exec-0", dataContext0.createExecutorInfo(SORT_MANAGER)); FetchResult execFetch = fetchBlocks("exec-0", new String[]{"shuffle_1_0_0", "shuffle_1_0_1"}, clientConf, 1 /* port */); assertTrue(execFetch.successBlocks.isEmpty()); assertEquals(Sets.newHashSet("shuffle_1_0_0", "shuffle_1_0_1"), execFetch.failedBlocks); }
@BeforeClass public static void beforeAll() throws IOException { Random rand = new Random(); for (byte[] block : exec0Blocks) { rand.nextBytes(block); } for (byte[] block: exec1Blocks) { rand.nextBytes(block); } dataContext0 = new TestShuffleDataContext(2, 5); dataContext0.create(); dataContext0.insertSortShuffleData(0, 0, exec0Blocks); conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); handler = new ExternalShuffleBlockHandler(conf, null); TransportContext transportContext = new TransportContext(conf, handler); server = transportContext.createServer(); }
void createClient(String secret, boolean enableAes) throws Exception { TransportConf clientConf = enableAes ? conf : new TransportConf("rpc", MapConfigProvider.EMPTY); List<TransportClientBootstrap> bootstraps = Arrays.asList( new AuthClientBootstrap(clientConf, appId, createKeyHolder(secret))); this.client = ctx.createClientFactory(bootstraps) .createClient(TestUtils.getLocalHost(), server.getPort()); }
void createClient(String secret, boolean enableAes) throws Exception { TransportConf clientConf = enableAes ? conf : new TransportConf("rpc", MapConfigProvider.EMPTY); List<TransportClientBootstrap> bootstraps = Arrays.asList( new AuthClientBootstrap(clientConf, appId, createKeyHolder(secret))); this.client = ctx.createClientFactory(bootstraps) .createClient(TestUtils.getLocalHost(), server.getPort()); }
@BeforeClass public static void beforeAll() throws IOException { Random rand = new Random(); for (byte[] block : exec0Blocks) { rand.nextBytes(block); } for (byte[] block: exec1Blocks) { rand.nextBytes(block); } dataContext0 = new TestShuffleDataContext(2, 5); dataContext0.create(); dataContext0.insertSortShuffleData(0, 0, exec0Blocks); conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); handler = new ExternalShuffleBlockHandler(conf, null); TransportContext transportContext = new TransportContext(conf, handler); server = transportContext.createServer(); }
@BeforeClass public static void beforeAll() throws IOException { Random rand = new Random(); for (byte[] block : exec0Blocks) { rand.nextBytes(block); } for (byte[] block: exec1Blocks) { rand.nextBytes(block); } dataContext0 = new TestShuffleDataContext(2, 5); dataContext0.create(); dataContext0.insertSortShuffleData(0, 0, exec0Blocks); conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); handler = new ExternalShuffleBlockHandler(conf, null); TransportContext transportContext = new TransportContext(conf, handler); server = transportContext.createServer(); }
@Before public void setUp() { conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); RpcHandler rpcHandler = new NoOpRpcHandler(); context = new TransportContext(conf, rpcHandler); server1 = context.createServer(); server2 = context.createServer(); }
@Before public void setUp() { conf = new TransportConf("shuffle", MapConfigProvider.EMPTY); RpcHandler rpcHandler = new NoOpRpcHandler(); context = new TransportContext(conf, rpcHandler); server1 = context.createServer(); server2 = context.createServer(); }
/** Creates an ExternalShuffleClient and attempts to register with the server. */ private void validate(String appId, String secretKey, boolean encrypt) throws IOException, InterruptedException { TransportConf testConf = conf; if (encrypt) { testConf = new TransportConf("shuffle", new MapConfigProvider( ImmutableMap.of("spark.authenticate.enableSaslEncryption", "true"))); } ExternalShuffleClient client = new ExternalShuffleClient(testConf, new TestSecretKeyHolder(appId, secretKey), true, 5000); client.init(appId); // Registration either succeeds or throws an exception. client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), "exec0", new ExecutorShuffleInfo(new String[0], 0, "org.apache.spark.shuffle.sort.SortShuffleManager")); client.close(); }
/** Creates an ExternalShuffleClient and attempts to register with the server. */ private void validate(String appId, String secretKey, boolean encrypt) throws IOException, InterruptedException { TransportConf testConf = conf; if (encrypt) { testConf = new TransportConf("shuffle", new MapConfigProvider( ImmutableMap.of("spark.authenticate.enableSaslEncryption", "true"))); } ExternalShuffleClient client = new ExternalShuffleClient(testConf, new TestSecretKeyHolder(appId, secretKey), true); client.init(appId); // Registration either succeeds or throws an exception. client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), "exec0", new ExecutorShuffleInfo(new String[0], 0, "org.apache.spark.shuffle.sort.SortShuffleManager")); client.close(); }