public ExternalShuffleBlockHandler(TransportConf conf, File registeredExecutorFile) throws IOException { this(new OneForOneStreamManager(), new ExternalShuffleBlockResolver(conf, registeredExecutorFile)); }
/** * Removes an application (once it has been terminated), and optionally will clean up any * local directories associated with the executors of that application in a separate thread. */ public void applicationRemoved(String appId, boolean cleanupLocalDirs) { blockManager.applicationRemoved(appId, cleanupLocalDirs); }
@Override public ManagedBuffer next() { final ManagedBuffer block = blockManager.getBlockData(appId, execId, shuffleId, mapIdAndReduceIds[index], mapIdAndReduceIds[index + 1]); index += 2; metrics.blockTransferRateBytes.mark(block != null ? block.size() : 0); return block; } }
@Test public void noCleanupAndCleanup() throws IOException { TestShuffleDataContext dataContext = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", false /* cleanup */); assertStillThere(dataContext); resolver.registerExecutor("app", "exec1", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true /* cleanup */); assertCleanedUp(dataContext); }
private void cleanupOnlyRegisteredExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec1", "app"); assertStillThere(dataContext); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
@Test public void testSortShuffleBlocks() throws IOException { ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null); resolver.registerExecutor("app0", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); InputStream block0Stream = resolver.getBlockData("app0", "exec0", 0, 0, 0).createInputStream(); String block0 = CharStreams.toString( new InputStreamReader(block0Stream, StandardCharsets.UTF_8)); block0Stream.close(); assertEquals(sortBlock0, block0); InputStream block1Stream = resolver.getBlockData("app0", "exec0", 0, 0, 1).createInputStream(); String block1 = CharStreams.toString( new InputStreamReader(block1Stream, StandardCharsets.UTF_8)); block1Stream.close(); assertEquals(sortBlock1, block1); }
/** * Register an (application, executor) with the given shuffle info. * * The "re-" is meant to highlight the intended use of this method -- when this service is * restarted, this is used to restore the state of executors from before the restart. Normal * registration will happen via a message handled in receive() * * @param appExecId * @param executorInfo */ public void reregisterExecutor(AppExecId appExecId, ExecutorShuffleInfo executorInfo) { blockManager.registerExecutor(appExecId.appId, appExecId.execId, executorInfo); }
/** Creates spill file(s) within the local dirs. */ public void insertSpillData() throws IOException { String filename = "temp_local_" + UUID.randomUUID(); OutputStream dataStream = null; try { dataStream = new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, filename)); dataStream.write(42); } finally { Closeables.close(dataStream, false); } }
if (db != null) { try { db.delete(dbAppExecKey(fullId)); } catch (IOException e) { logger.error("Error deleting {} from executor state db", appId, e); directoryCleaner.execute(() -> deleteExecutorDirs(executor.localDirs));
/** * Hashes a filename into the corresponding local directory, in a manner consistent with * Spark's DiskBlockManager.getFile(). */ @VisibleForTesting static File getFile(String[] localDirs, int subDirsPerLocalDir, String filename) { int hash = JavaUtils.nonNegativeHash(filename); String localDir = localDirs[hash % localDirs.length]; int subDirId = (hash / localDirs.length) % subDirsPerLocalDir; return new File(createNormalizedInternedPathname( localDir, String.format("%02x", subDirId), filename)); }
/** * Clean up any non-shuffle files in any local directories associated with an finished executor. */ public void executorRemoved(String executorId, String appId) { blockManager.executorRemoved(executorId, appId); }
public void close() { blockManager.close(); }
/** Registers a new Executor with all the configuration we need to find its shuffle files. */ public void registerExecutor( String appId, String execId, ExecutorShuffleInfo executorInfo) { AppExecId fullId = new AppExecId(appId, execId); logger.info("Registered executor {} with {}", fullId, executorInfo); if (!knownManagers.contains(executorInfo.shuffleManager)) { throw new UnsupportedOperationException( "Unsupported shuffle manager of executor: " + executorInfo); } try { if (db != null) { byte[] key = dbAppExecKey(fullId); byte[] value = mapper.writeValueAsString(executorInfo).getBytes(StandardCharsets.UTF_8); db.put(key, value); } } catch (Exception e) { logger.error("Error saving registered executors", e); } executors.put(fullId, executorInfo); }
@Test public void noCleanupAndCleanup() throws IOException { TestShuffleDataContext dataContext = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", false /* cleanup */); assertStillThere(dataContext); resolver.registerExecutor("app", "exec1", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true /* cleanup */); assertCleanedUp(dataContext); }
private void cleanupOnRemovedExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
@Test public void testSortShuffleBlocks() throws IOException { ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null); resolver.registerExecutor("app0", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); InputStream block0Stream = resolver.getBlockData("app0", "exec0", 0, 0, 0).createInputStream(); String block0 = CharStreams.toString( new InputStreamReader(block0Stream, StandardCharsets.UTF_8)); block0Stream.close(); assertEquals(sortBlock0, block0); InputStream block1Stream = resolver.getBlockData("app0", "exec0", 0, 0, 1).createInputStream(); String block1 = CharStreams.toString( new InputStreamReader(block1Stream, StandardCharsets.UTF_8)); block1Stream.close(); assertEquals(sortBlock1, block1); }
/** * Register an (application, executor) with the given shuffle info. * * The "re-" is meant to highlight the intended use of this method -- when this service is * restarted, this is used to restore the state of executors from before the restart. Normal * registration will happen via a message handled in receive() * * @param appExecId * @param executorInfo */ public void reregisterExecutor(AppExecId appExecId, ExecutorShuffleInfo executorInfo) { blockManager.registerExecutor(appExecId.appId, appExecId.execId, executorInfo); }
/** Creates spill file(s) within the local dirs. */ public void insertSpillData() throws IOException { String filename = "temp_local_" + UUID.randomUUID(); OutputStream dataStream = null; try { dataStream = new FileOutputStream( ExternalShuffleBlockResolver.getFile(localDirs, subDirsPerLocalDir, filename)); dataStream.write(42); } finally { Closeables.close(dataStream, false); } }
if (db != null) { try { db.delete(dbAppExecKey(fullId)); } catch (IOException e) { logger.error("Error deleting {} from executor state db", appId, e); directoryCleaner.execute(() -> deleteExecutorDirs(executor.localDirs));
/** * Hashes a filename into the corresponding local directory, in a manner consistent with * Spark's DiskBlockManager.getFile(). */ @VisibleForTesting static File getFile(String[] localDirs, int subDirsPerLocalDir, String filename) { int hash = JavaUtils.nonNegativeHash(filename); String localDir = localDirs[hash % localDirs.length]; int subDirId = (hash / localDirs.length) % subDirsPerLocalDir; return new File(createNormalizedInternedPathname( localDir, String.format("%02x", subDirId), filename)); }