private static void assertCleanedUp(TestShuffleDataContext dataContext) { for (String localDir : dataContext.localDirs) { File[] dirs = new File[] {new File(localDir)}; assertOnlyShuffleDataInDir(dirs); } }
private static TestShuffleDataContext initDataContextWithShuffleFiles() throws IOException { TestShuffleDataContext dataContext = createDataContext(); createShuffleFiles(dataContext); createNonShuffleFiles(dataContext); return dataContext; }
private static TestShuffleDataContext initDataContextWithoutShuffleFiles() throws IOException { TestShuffleDataContext dataContext = createDataContext(); createNonShuffleFiles(dataContext); return dataContext; }
private void cleanupOnlyRemovedExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext0 = initDataContext(withShuffleFiles); TestShuffleDataContext dataContext1 = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext0.createExecutorInfo(SORT_MANAGER)); resolver.registerExecutor("app", "exec1", dataContext1.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec-nonexistent", "app"); assertStillThere(dataContext0); assertStillThere(dataContext1); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext0); assertStillThere(dataContext1); resolver.executorRemoved("exec1", "app"); assertCleanedUp(dataContext0); assertCleanedUp(dataContext1); // Make sure it's not an error to cleanup multiple times resolver.executorRemoved("exec1", "app"); assertCleanedUp(dataContext0); assertCleanedUp(dataContext1); }
private void cleanupOnRemovedExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
private void cleanupUsesExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); AtomicBoolean cleanupCalled = new AtomicBoolean(false); // Executor which does nothing to ensure we're actually using it. Executor noThreadExecutor = runnable -> cleanupCalled.set(true); ExternalShuffleBlockResolver manager = new ExternalShuffleBlockResolver(conf, null, noThreadExecutor); manager.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); manager.executorRemoved("exec0", "app"); assertTrue(cleanupCalled.get()); assertStillThere(dataContext); }
@Test public void cleanupUsesExecutorWithShuffleFiles() throws IOException { cleanupUsesExecutor(true); }
@Test public void cleanupOnlyRemovedExecutorWithShuffleFiles() throws IOException { cleanupOnlyRemovedExecutor(true); }
@Test public void cleanupOnlyRegisteredExecutorWithShuffleFiles() throws IOException { cleanupOnlyRegisteredExecutor(true); }
@Test public void cleanupOnRemovedExecutorWithoutShuffleFiles() throws IOException { cleanupOnRemovedExecutor(false); }
private void cleanupOnlyRemovedExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext0 = initDataContext(withShuffleFiles); TestShuffleDataContext dataContext1 = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext0.createExecutorInfo(SORT_MANAGER)); resolver.registerExecutor("app", "exec1", dataContext1.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec-nonexistent", "app"); assertStillThere(dataContext0); assertStillThere(dataContext1); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext0); assertStillThere(dataContext1); resolver.executorRemoved("exec1", "app"); assertCleanedUp(dataContext0); assertCleanedUp(dataContext1); // Make sure it's not an error to cleanup multiple times resolver.executorRemoved("exec1", "app"); assertCleanedUp(dataContext0); assertCleanedUp(dataContext1); }
private static TestShuffleDataContext initDataContextWithoutShuffleFiles() throws IOException { TestShuffleDataContext dataContext = createDataContext(); createNonShuffleFiles(dataContext); return dataContext; }
private void cleanupOnRemovedExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
private void cleanupUsesExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); AtomicBoolean cleanupCalled = new AtomicBoolean(false); // Executor which does nothing to ensure we're actually using it. Executor noThreadExecutor = runnable -> cleanupCalled.set(true); ExternalShuffleBlockResolver manager = new ExternalShuffleBlockResolver(conf, null, noThreadExecutor); manager.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); manager.executorRemoved("exec0", "app"); assertTrue(cleanupCalled.get()); assertStillThere(dataContext); }
@Test public void cleanupUsesExecutorWithoutShuffleFiles() throws IOException { cleanupUsesExecutor(false); }
@Test public void cleanupOnlyRemovedExecutorWithShuffleFiles() throws IOException { cleanupOnlyRemovedExecutor(true); }
@Test public void cleanupOnlyRegisteredExecutorWithoutShuffleFiles() throws IOException { cleanupOnlyRegisteredExecutor(false); }
@Test public void cleanupOnRemovedExecutorWithoutShuffleFiles() throws IOException { cleanupOnRemovedExecutor(false); }
private void cleanupOnlyRegisteredExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec1", "app"); assertStillThere(dataContext); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
private static TestShuffleDataContext initDataContextWithShuffleFiles() throws IOException { TestShuffleDataContext dataContext = createDataContext(); createShuffleFiles(dataContext); createNonShuffleFiles(dataContext); return dataContext; }