/** Registers a new Executor with all the configuration we need to find its shuffle files. */ public void registerExecutor( String appId, String execId, ExecutorShuffleInfo executorInfo) { AppExecId fullId = new AppExecId(appId, execId); logger.info("Registered executor {} with {}", fullId, executorInfo); if (!knownManagers.contains(executorInfo.shuffleManager)) { throw new UnsupportedOperationException( "Unsupported shuffle manager of executor: " + executorInfo); } try { if (db != null) { byte[] key = dbAppExecKey(fullId); byte[] value = mapper.writeValueAsString(executorInfo).getBytes(StandardCharsets.UTF_8); db.put(key, value); } } catch (Exception e) { logger.error("Error saving registered executors", e); } executors.put(fullId, executorInfo); }
/** * Obtains a FileSegmentManagedBuffer from a shuffle block id. We expect the blockId has the * format "shuffle_ShuffleId_MapId_ReduceId" (from ShuffleBlockId), and additionally make * assumptions about how the hash and sort based shuffles store their data. */ public ManagedBuffer getBlockData(String appId, String execId, String blockId) { String[] blockIdParts = blockId.split("_"); if (blockIdParts.length < 4) { throw new IllegalArgumentException("Unexpected block id format: " + blockId); } else if (!blockIdParts[0].equals("shuffle")) { throw new IllegalArgumentException("Expected shuffle block id, got: " + blockId); } int shuffleId = Integer.parseInt(blockIdParts[1]); int mapId = Integer.parseInt(blockIdParts[2]); int reduceId = Integer.parseInt(blockIdParts[3]); ExecutorShuffleInfo executor = executors.get(new AppExecId(appId, execId)); if (executor == null) { throw new RuntimeException( String.format("Executor is not registered (appId=%s, execId=%s)", appId, execId)); } return getSortBasedShuffleBlockData(executor, shuffleId, mapId, reduceId); }
@VisibleForTesting static ConcurrentMap<AppExecId, ExecutorShuffleInfo> reloadRegisteredExecutors(DB db) throws IOException { ConcurrentMap<AppExecId, ExecutorShuffleInfo> registeredExecutors = Maps.newConcurrentMap(); if (db != null) { DBIterator itr = db.iterator(); itr.seek(APP_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_KEY_PREFIX)) { break; } AppExecId id = parseDbAppExecKey(key); logger.info("Reloading registered executors: " + id.toString()); ExecutorShuffleInfo shuffleInfo = mapper.readValue(e.getValue(), ExecutorShuffleInfo.class); registeredExecutors.put(id, shuffleInfo); } } return registeredExecutors; } }
@VisibleForTesting static ConcurrentMap<AppExecId, ExecutorShuffleInfo> reloadRegisteredExecutors(DB db) throws IOException { ConcurrentMap<AppExecId, ExecutorShuffleInfo> registeredExecutors = Maps.newConcurrentMap(); if (db != null) { DBIterator itr = db.iterator(); itr.seek(APP_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_KEY_PREFIX)) { break; } AppExecId id = parseDbAppExecKey(key); logger.info("Reloading registered executors: " + id.toString()); ExecutorShuffleInfo shuffleInfo = mapper.readValue(e.getValue(), ExecutorShuffleInfo.class); registeredExecutors.put(id, shuffleInfo); } } return registeredExecutors; } }
@VisibleForTesting static ConcurrentMap<AppExecId, ExecutorShuffleInfo> reloadRegisteredExecutors(DB db) throws IOException { ConcurrentMap<AppExecId, ExecutorShuffleInfo> registeredExecutors = Maps.newConcurrentMap(); if (db != null) { DBIterator itr = db.iterator(); itr.seek(APP_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_KEY_PREFIX)) { break; } AppExecId id = parseDbAppExecKey(key); logger.info("Reloading registered executors: " + id.toString()); ExecutorShuffleInfo shuffleInfo = mapper.readValue(e.getValue(), ExecutorShuffleInfo.class); registeredExecutors.put(id, shuffleInfo); } } return registeredExecutors; } }
@Test public void jsonSerializationOfExecutorRegistration() throws IOException { ObjectMapper mapper = new ObjectMapper(); AppExecId appId = new AppExecId("foo", "bar"); String appIdJson = mapper.writeValueAsString(appId); AppExecId parsedAppId = mapper.readValue(appIdJson, AppExecId.class); assertEquals(parsedAppId, appId); ExecutorShuffleInfo shuffleInfo = new ExecutorShuffleInfo(new String[]{"/bippy", "/flippy"}, 7, SORT_MANAGER); String shuffleJson = mapper.writeValueAsString(shuffleInfo); ExecutorShuffleInfo parsedShuffleInfo = mapper.readValue(shuffleJson, ExecutorShuffleInfo.class); assertEquals(parsedShuffleInfo, shuffleInfo); // Intentionally keep these hard-coded strings in here, to check backwards-compatibility. // its not legacy yet, but keeping this here in case anybody changes it String legacyAppIdJson = "{\"appId\":\"foo\", \"execId\":\"bar\"}"; assertEquals(appId, mapper.readValue(legacyAppIdJson, AppExecId.class)); String legacyShuffleJson = "{\"localDirs\": [\"/bippy\", \"/flippy\"], " + "\"subDirsPerLocalDir\": 7, \"shuffleManager\": " + "\"" + SORT_MANAGER + "\"}"; assertEquals(shuffleInfo, mapper.readValue(legacyShuffleJson, ExecutorShuffleInfo.class)); }
/** Registers a new Executor with all the configuration we need to find its shuffle files. */ public void registerExecutor( String appId, String execId, ExecutorShuffleInfo executorInfo) { AppExecId fullId = new AppExecId(appId, execId); logger.info("Registered executor {} with {}", fullId, executorInfo); if (!knownManagers.contains(executorInfo.shuffleManager)) { throw new UnsupportedOperationException( "Unsupported shuffle manager of executor: " + executorInfo); } try { if (db != null) { byte[] key = dbAppExecKey(fullId); byte[] value = mapper.writeValueAsString(executorInfo).getBytes(StandardCharsets.UTF_8); db.put(key, value); } } catch (Exception e) { logger.error("Error saving registered executors", e); } executors.put(fullId, executorInfo); }
/** Registers a new Executor with all the configuration we need to find its shuffle files. */ public void registerExecutor( String appId, String execId, ExecutorShuffleInfo executorInfo) { AppExecId fullId = new AppExecId(appId, execId); logger.info("Registered executor {} with {}", fullId, executorInfo); if (!knownManagers.contains(executorInfo.shuffleManager)) { throw new UnsupportedOperationException( "Unsupported shuffle manager of executor: " + executorInfo); } try { if (db != null) { byte[] key = dbAppExecKey(fullId); byte[] value = mapper.writeValueAsString(executorInfo).getBytes(StandardCharsets.UTF_8); db.put(key, value); } } catch (Exception e) { logger.error("Error saving registered executors", e); } executors.put(fullId, executorInfo); }
/** * Removes all the non-shuffle files in any local directories associated with the finished * executor. */ public void executorRemoved(String executorId, String appId) { logger.info("Clean up non-shuffle files associated with the finished executor {}", executorId); AppExecId fullId = new AppExecId(appId, executorId); final ExecutorShuffleInfo executor = executors.get(fullId); if (executor == null) { // Executor not registered, skip clean up of the local directories. logger.info("Executor is not registered (appId={}, execId={})", appId, executorId); } else { logger.info("Cleaning up non-shuffle files in executor {}'s {} local dirs", fullId, executor.localDirs.length); // Execute the actual deletion in a different thread, as it may take some time. directoryCleaner.execute(() -> deleteNonShuffleFiles(executor.localDirs)); } }
@Test public void jsonSerializationOfExecutorRegistration() throws IOException { ObjectMapper mapper = new ObjectMapper(); AppExecId appId = new AppExecId("foo", "bar"); String appIdJson = mapper.writeValueAsString(appId); AppExecId parsedAppId = mapper.readValue(appIdJson, AppExecId.class); assertEquals(parsedAppId, appId); ExecutorShuffleInfo shuffleInfo = new ExecutorShuffleInfo(new String[]{"/bippy", "/flippy"}, 7, SORT_MANAGER); String shuffleJson = mapper.writeValueAsString(shuffleInfo); ExecutorShuffleInfo parsedShuffleInfo = mapper.readValue(shuffleJson, ExecutorShuffleInfo.class); assertEquals(parsedShuffleInfo, shuffleInfo); // Intentionally keep these hard-coded strings in here, to check backwards-compatability. // its not legacy yet, but keeping this here in case anybody changes it String legacyAppIdJson = "{\"appId\":\"foo\", \"execId\":\"bar\"}"; assertEquals(appId, mapper.readValue(legacyAppIdJson, AppExecId.class)); String legacyShuffleJson = "{\"localDirs\": [\"/bippy\", \"/flippy\"], " + "\"subDirsPerLocalDir\": 7, \"shuffleManager\": " + "\"" + SORT_MANAGER + "\"}"; assertEquals(shuffleInfo, mapper.readValue(legacyShuffleJson, ExecutorShuffleInfo.class)); } }
/** * Obtains a FileSegmentManagedBuffer from (shuffleId, mapId, reduceId). We make assumptions * about how the hash and sort based shuffles store their data. */ public ManagedBuffer getBlockData( String appId, String execId, int shuffleId, int mapId, int reduceId) { ExecutorShuffleInfo executor = executors.get(new AppExecId(appId, execId)); if (executor == null) { throw new RuntimeException( String.format("Executor is not registered (appId=%s, execId=%s)", appId, execId)); } return getSortBasedShuffleBlockData(executor, shuffleId, mapId, reduceId); }
@Test public void jsonSerializationOfExecutorRegistration() throws IOException { ObjectMapper mapper = new ObjectMapper(); AppExecId appId = new AppExecId("foo", "bar"); String appIdJson = mapper.writeValueAsString(appId); AppExecId parsedAppId = mapper.readValue(appIdJson, AppExecId.class); assertEquals(parsedAppId, appId); ExecutorShuffleInfo shuffleInfo = new ExecutorShuffleInfo(new String[]{"/bippy", "/flippy"}, 7, SORT_MANAGER); String shuffleJson = mapper.writeValueAsString(shuffleInfo); ExecutorShuffleInfo parsedShuffleInfo = mapper.readValue(shuffleJson, ExecutorShuffleInfo.class); assertEquals(parsedShuffleInfo, shuffleInfo); // Intentionally keep these hard-coded strings in here, to check backwards-compatibility. // its not legacy yet, but keeping this here in case anybody changes it String legacyAppIdJson = "{\"appId\":\"foo\", \"execId\":\"bar\"}"; assertEquals(appId, mapper.readValue(legacyAppIdJson, AppExecId.class)); String legacyShuffleJson = "{\"localDirs\": [\"/bippy\", \"/flippy\"], " + "\"subDirsPerLocalDir\": 7, \"shuffleManager\": " + "\"" + SORT_MANAGER + "\"}"; assertEquals(shuffleInfo, mapper.readValue(legacyShuffleJson, ExecutorShuffleInfo.class)); }
/** * Removes all the non-shuffle files in any local directories associated with the finished * executor. */ public void executorRemoved(String executorId, String appId) { logger.info("Clean up non-shuffle files associated with the finished executor {}", executorId); AppExecId fullId = new AppExecId(appId, executorId); final ExecutorShuffleInfo executor = executors.get(fullId); if (executor == null) { // Executor not registered, skip clean up of the local directories. logger.info("Executor is not registered (appId={}, execId={})", appId, executorId); } else { logger.info("Cleaning up non-shuffle files in executor {}'s {} local dirs", fullId, executor.localDirs.length); // Execute the actual deletion in a different thread, as it may take some time. directoryCleaner.execute(() -> deleteNonShuffleFiles(executor.localDirs)); } }
/** * Obtains a FileSegmentManagedBuffer from (shuffleId, mapId, reduceId). We make assumptions * about how the hash and sort based shuffles store their data. */ public ManagedBuffer getBlockData( String appId, String execId, int shuffleId, int mapId, int reduceId) { ExecutorShuffleInfo executor = executors.get(new AppExecId(appId, execId)); if (executor == null) { throw new RuntimeException( String.format("Executor is not registered (appId=%s, execId=%s)", appId, execId)); } return getSortBasedShuffleBlockData(executor, shuffleId, mapId, reduceId); }