@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testResetDeleteWhenBuildingHoldingCleanupTracker() throws Exception { createProcExecutor(); ExchangeProcedure proc1 = new ExchangeProcedure(); ExchangeProcedure proc2 = new ExchangeProcedure(); procExecutor.submitProcedure(proc1); long procId2 = procExecutor.submitProcedure(proc2); Thread.sleep(500); procStore.rollWriterForTesting(); proc1.exchanger.exchange(Boolean.TRUE); Thread.sleep(500); FileStatus[] walFiles = fs.listStatus(logDir); Arrays.sort(walFiles, (f1, f2) -> f1.getPath().getName().compareTo(f2.getPath().getName())); // corrupt the first proc wal file, so we will have a partial tracker for it after restarting corrupt(walFiles[0]); ProcedureTestingUtility.restart(procExecutor, false, true); // also update proc2, which means that all the procedures in the first proc wal have been // updated and it should be deleted. proc2 = (ExchangeProcedure) procExecutor.getProcedure(procId2); proc2.exchanger.exchange(Boolean.TRUE); htu.waitFor(10000, () -> !fs.exists(walFiles[0].getPath())); }
store.rollWriterForTesting(); System.out.println("Starting new log : " + store.getActiveLogs().get(store.getActiveLogs().size() - 1));
private void writeWals() throws IOException { List<Integer> procStates = shuffleProcWriteSequence(); TestProcedure[] procs = new TestProcedure[numProcs + 1]; // 0 is not used. int numProcsPerWal = numWals > 0 ? procStates.size() / numWals : Integer.MAX_VALUE; long startTime = currentTimeMillis(); long lastTime = startTime; for (int i = 0; i < procStates.size(); ++i) { int procId = procStates.get(i); if (procId < 0) { store.delete(procs[-procId].getProcId()); procs[-procId] = null; } else if (procs[procId] == null) { procs[procId] = new TestProcedure(procId, 0); procs[procId].setData(serializedState); store.insert(procs[procId], null); } else { store.update(procs[procId]); } if (i > 0 && i % numProcsPerWal == 0) { long currentTime = currentTimeMillis(); System.out.println("Forcing wall roll. Time taken on last WAL: " + (currentTime - lastTime) / 1000.0f + " sec"); store.rollWriterForTesting(); lastTime = currentTime; } } long timeTaken = currentTimeMillis() - startTime; System.out.println("\n\nDone writing WALs.\nNum procs : " + numProcs + "\nTotal time taken : " + StringUtils.humanTimeDiff(timeTaken) + "\n\n"); }
@Test public void testProcedureShouldNotCleanOnLoad() throws Exception { createProcExecutor(); final RootProcedure proc = new RootProcedure(); long rootProc = procExecutor.submitProcedure(proc); LOG.info("Begin to execute " + rootProc); // wait until the child procedure arrival htu.waitFor(10000, () -> procExecutor.getProcedures().size() >= 2); SuspendProcedure suspendProcedure = (SuspendProcedure) procExecutor .getProcedures().get(1); // wait until the suspendProcedure executed suspendProcedure.latch.countDown(); Thread.sleep(100); // roll the procedure log LOG.info("Begin to roll log "); procStore.rollWriterForTesting(); LOG.info("finish to roll log "); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); assertTrue(procExecutor.getProcedure(rootProc) != null); Thread.sleep(500); LOG.info("begin to restart2 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart2 "); assertTrue(procExecutor.getProcedure(rootProc) != null); }
@Test public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); // generates first log with p1 + p2 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); procStore.update(p2); procStore.rollWriterForTesting(); // generates second log with p2 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); procStore.update(p2); procStore.rollWriterForTesting(); // generates third log with p2 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. assertEquals(4, procStore.getActiveLogs().size()); procStore.update(p1); procStore.rollWriterForTesting(); // generates fourth log with p1 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. assertEquals(3, procStore.getActiveLogs().size()); assertFalse(procStore.getActiveLogs().contains(log1)); assertFalse(procStore.getActiveLogs().contains(log2)); }
@Test public void testWalCleanerWithEmptyRolls() throws Exception { final Procedure<?>[] procs = new Procedure[3]; for (int i = 0; i < procs.length; ++i) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); } assertEquals(1, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(3, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.update(procs[i]); procStore.rollWriterForTesting(); procStore.rollWriterForTesting(); if (i < (procs.length - 1)) { assertEquals(3 + ((i + 1) * 2), procStore.getActiveLogs().size()); } } assertEquals(7, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.delete(procs[i].getProcId()); assertEquals(7 - ((i + 1) * 2), procStore.getActiveLogs().size()); } assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testWalCleanerSequentialClean() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(logs.size(), i + 2); // Extra 1 for current ongoing wal. } // Delete procedures in sequential order make sure that only the corresponding wal is deleted // from logs list. final int[] deleteOrder = new int[] { 0, 1, 2, 3, 4 }; for (int i = 0; i < deleteOrder.length; i++) { procStore.delete(procs[deleteOrder[i]].getProcId()); procStore.removeInactiveLogsForTesting(); assertFalse(logs.get(deleteOrder[i]).toString(), procStore.getActiveLogs().contains(logs.get(deleteOrder[i]))); assertEquals(procStore.getActiveLogs().size(), procs.length - i); } }
@Test public void testWalCleanerNoHoles() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. } for (int i = 1; i < procs.length; i++) { procStore.delete(procs[i].getProcId()); } assertEquals(procs.length + 1, procStore.getActiveLogs().size()); procStore.delete(procs[0].getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testRollAndRemove() throws IOException { // Insert something in the log Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // everything will be up to date in the second log // so we can remove the first one procStore.update(proc1); procStore.update(proc2); assertEquals(1, procStore.getActiveLogs().size()); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // remove everything active // so we can remove all the logs procStore.delete(proc1.getProcId()); procStore.delete(proc2.getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
assertEquals(1, STORE.getActiveLogs().size()); for (int i = 0; i < WAL_COUNT - 1; i++) { assertTrue(STORE.rollWriterForTesting()); Thread.sleep(1000); STORE.rollWriterForTesting();
@Test public void testCompletedProcedure() throws InterruptedException, IOException { long procId = EXEC.submitProcedure(new ExchangeProcedure()); EXCHANGER.exchange(Boolean.FALSE); UTIL.waitFor(10000, () -> EXEC.isFinished(procId)); for (int i = 0; i < WAL_COUNT - 1; i++) { assertTrue(STORE.rollWriterForTesting()); // The exchange procedure is completed but still not deleted yet so we can not delete the // oldest wal file long pid = EXEC.submitProcedure(new NoopNoAckProcedure()); assertEquals(2 + i, STORE.getActiveLogs().size()); UTIL.waitFor(10000, () -> EXEC.isFinished(pid)); } // Only the exchange procedure can not be deleted UTIL.waitFor(10000, () -> EXEC.getCompletedSize() == 1); STORE.rollWriterForTesting(); UTIL.waitFor(10000, () -> STORE.getActiveLogs().size() <= 1); } }
procStore.insert(procs[3], null); procStore.delete(procs[0].getProcId()); procStore.rollWriterForTesting(); procStore.delete(procs[2].getProcId()); procStore.update(procs[3]); procStore.insert(procs[4], null); procStore.rollWriterForTesting(); procStore.delete(procs[4].getProcId()); procStore.insert(procs[5], null);
Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); procStore.rollWriterForTesting(); procStore.rollWriterForTesting(); procStore.rollWriterForTesting();
procStore.insert(procs[i], null); procStore.rollWriterForTesting(); for (int i = 0; i < procs.length; ++i) { procStore.update(procs[i]); procStore.rollWriterForTesting();
procStore.rollWriterForTesting(); for (int i = 1; i <= rootProcs.length; i++) { TestProcedure b = new TestProcedure(rootProcs.length + i, i); procStore.rollWriterForTesting(); for (int i = 1; i <= rootProcs.length; i++) { procStore.update(new TestProcedure(rootProcs.length + i, i));
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testWalCleanerNoHoles() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. } for (int i = 1; i < procs.length; i++) { procStore.delete(procs[i].getProcId()); } assertEquals(procs.length + 1, procStore.getActiveLogs().size()); procStore.delete(procs[0].getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testCompletedProcedure() throws InterruptedException, IOException { long procId = EXEC.submitProcedure(new ExchangeProcedure()); EXCHANGER.exchange(Boolean.FALSE); UTIL.waitFor(10000, () -> EXEC.isFinished(procId)); for (int i = 0; i < WAL_COUNT - 1; i++) { assertTrue(STORE.rollWriterForTesting()); // The exchange procedure is completed but still not deleted yet so we can not delete the // oldest wal file long pid = EXEC.submitProcedure(new NoopNoAckProcedure()); assertEquals(2 + i, STORE.getActiveLogs().size()); UTIL.waitFor(10000, () -> EXEC.isFinished(pid)); } // Only the exchange procedure can not be deleted UTIL.waitFor(10000, () -> EXEC.getCompletedSize() == 1); STORE.rollWriterForTesting(); UTIL.waitFor(10000, () -> STORE.getActiveLogs().size() <= 1); } }