@Override public void close() { heartbeaterThread.shutdownNow(); super.close(); }
@Override public void close() { checkInit(); if (clientFactory != null) { clientFactory.close(); clientFactory = null; } } }
/** Creates an ExternalShuffleClient and attempts to register with the server. */ private void validate(String appId, String secretKey, boolean encrypt) throws IOException, InterruptedException { TransportConf testConf = conf; if (encrypt) { testConf = new TransportConf("shuffle", new MapConfigProvider( ImmutableMap.of("spark.authenticate.enableSaslEncryption", "true"))); } ExternalShuffleClient client = new ExternalShuffleClient(testConf, new TestSecretKeyHolder(appId, secretKey), true, 5000); client.init(appId); // Registration either succeeds or throws an exception. client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), "exec0", new ExecutorShuffleInfo(new String[0], 0, "org.apache.spark.shuffle.sort.SortShuffleManager")); client.close(); }
ExternalShuffleClient client = new ExternalShuffleClient(clientConf, null, false); client.init(APP_ID); client.fetchBlocks(TestUtils.getLocalHost(), port, execId, blockIds, new BlockFetchingListener() { @Override fail("Timeout getting response from the server"); client.close(); return res;
private static void registerExecutor(String executorId, ExecutorShuffleInfo executorInfo) throws IOException, InterruptedException { ExternalShuffleClient client = new ExternalShuffleClient(conf, null, false, 5000); client.init(APP_ID); client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), executorId, executorInfo); }
/** Creates an ExternalShuffleClient and attempts to register with the server. */ private void validate(String appId, String secretKey, boolean encrypt) throws IOException, InterruptedException { TransportConf testConf = conf; if (encrypt) { testConf = new TransportConf("shuffle", new MapConfigProvider( ImmutableMap.of("spark.authenticate.enableSaslEncryption", "true"))); } ExternalShuffleClient client = new ExternalShuffleClient(testConf, new TestSecretKeyHolder(appId, secretKey), true); client.init(appId); // Registration either succeeds or throws an exception. client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), "exec0", new ExecutorShuffleInfo(new String[0], 0, "org.apache.spark.shuffle.sort.SortShuffleManager")); client.close(); }
ExternalShuffleClient client = new ExternalShuffleClient(clientConf, null, false, 5000); client.init(APP_ID); client.fetchBlocks(TestUtils.getLocalHost(), port, execId, blockIds, new BlockFetchingListener() { @Override fail("Timeout getting response from the server"); client.close(); return res;
private static void registerExecutor(String executorId, ExecutorShuffleInfo executorInfo) throws IOException, InterruptedException { ExternalShuffleClient client = new ExternalShuffleClient(conf, null, false, 5000); client.init(APP_ID); client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), executorId, executorInfo); }
/** Creates an ExternalShuffleClient and attempts to register with the server. */ private void validate(String appId, String secretKey, boolean encrypt) throws IOException, InterruptedException { TransportConf testConf = conf; if (encrypt) { testConf = new TransportConf("shuffle", new MapConfigProvider( ImmutableMap.of("spark.authenticate.enableSaslEncryption", "true"))); } ExternalShuffleClient client = new ExternalShuffleClient(testConf, new TestSecretKeyHolder(appId, secretKey), true, 5000); client.init(appId); // Registration either succeeds or throws an exception. client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), "exec0", new ExecutorShuffleInfo(new String[0], 0, "org.apache.spark.shuffle.sort.SortShuffleManager")); client.close(); }
ExternalShuffleClient client = new ExternalShuffleClient(clientConf, null, false, 5000); client.init(APP_ID); client.fetchBlocks(TestUtils.getLocalHost(), port, execId, blockIds, new BlockFetchingListener() { @Override fail("Timeout getting response from the server"); client.close(); return res;
private static void registerExecutor(String executorId, ExecutorShuffleInfo executorInfo) throws IOException, InterruptedException { ExternalShuffleClient client = new ExternalShuffleClient(conf, null, false); client.init(APP_ID); client.registerWithShuffleServer(TestUtils.getLocalHost(), server.getPort(), executorId, executorInfo); }
@Override public void close() { heartbeaterThread.shutdownNow(); super.close(); }
@Override public MetricSet shuffleMetrics() { checkInit(); return clientFactory.getAllMetrics(); }
@Override public void close() { heartbeaterThread.shutdownNow(); super.close(); }
@Override public MetricSet shuffleMetrics() { checkInit(); return clientFactory.getAllMetrics(); }
@Override public void close() { checkInit(); if (clientFactory != null) { clientFactory.close(); clientFactory = null; } } }
/** * Registers this executor with an external shuffle server. This registration is required to * inform the shuffle server about where and how we store our shuffle files. * * @param host Host of shuffle server. * @param port Port of shuffle server. * @param execId This Executor's id. * @param executorInfo Contains all info necessary for the service to find our shuffle files. */ public void registerWithShuffleServer( String host, int port, String execId, ExecutorShuffleInfo executorInfo) throws IOException, InterruptedException { checkInit(); try (TransportClient client = clientFactory.createUnmanagedClient(host, port)) { ByteBuffer registerMessage = new RegisterExecutor(appId, execId, executorInfo).toByteBuffer(); client.sendRpcSync(registerMessage, 5000 /* timeoutMs */); } }
/** * Registers this executor with an external shuffle server. This registration is required to * inform the shuffle server about where and how we store our shuffle files. * * @param host Host of shuffle server. * @param port Port of shuffle server. * @param execId This Executor's id. * @param executorInfo Contains all info necessary for the service to find our shuffle files. */ public void registerWithShuffleServer( String host, int port, String execId, ExecutorShuffleInfo executorInfo) throws IOException, InterruptedException { checkInit(); try (TransportClient client = clientFactory.createUnmanagedClient(host, port)) { ByteBuffer registerMessage = new RegisterExecutor(appId, execId, executorInfo).toByteBuffer(); client.sendRpcSync(registerMessage, registrationTimeoutMs); } }
/** * Registers this executor with an external shuffle server. This registration is required to * inform the shuffle server about where and how we store our shuffle files. * * @param host Host of shuffle server. * @param port Port of shuffle server. * @param execId This Executor's id. * @param executorInfo Contains all info necessary for the service to find our shuffle files. */ public void registerWithShuffleServer( String host, int port, String execId, ExecutorShuffleInfo executorInfo) throws IOException, InterruptedException { checkInit(); try (TransportClient client = clientFactory.createUnmanagedClient(host, port)) { ByteBuffer registerMessage = new RegisterExecutor(appId, execId, executorInfo).toByteBuffer(); client.sendRpcSync(registerMessage, registrationTimeoutMs); } }
BlockFetchingListener listener, TempShuffleFileManager tempShuffleFileManager) { checkInit(); logger.debug("External shuffle fetch from {}:{} (executor id {})", host, port, execId); try {