private void startProcedureExecutor() throws IOException { procedureExecutor.startWorkers(); }
public static void initAndStartWorkers(ProcedureExecutor<?> procExecutor, int numThreads, boolean abortOnCorruption, boolean startWorkers) throws IOException { procExecutor.init(numThreads, abortOnCorruption); if (startWorkers) { procExecutor.startWorkers(); } }
@Test public void testTableProcedureSubProcedureDeadLock() throws Exception { // the shared procedure will also schedule a shared procedure, but after the exclusive procedure long procId1 = procExec.submitProcedure(new TableShardParentProcedure()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedure()); procExec.startWorkers(); UTIL.waitFor(10000, () -> procExec.getProcedures().stream().anyMatch(p -> p instanceof TableSharedProcedure)); procExec.getProcedures().stream().filter(p -> p instanceof TableSharedProcedure) .map(p -> (TableSharedProcedure) p).forEach(p -> p.latch.release()); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); } }
@Test public void testProcedureUpdatedShouldClean() throws Exception { createProcExecutor(); SuspendProcedure suspendProcedure = new SuspendProcedure(); long suspendProc = procExecutor.submitProcedure(suspendProcedure); LOG.info("Begin to execute " + suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(suspendProc) != null); // Wait until the suspendProc executed after restart suspendProcedure = (SuspendProcedure) procExecutor.getProcedure(suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
@Test public void testProcedureDeletedShouldClean() throws Exception { createProcExecutor(); WaitProcedure waitProcedure = new WaitProcedure(); long waitProce = procExecutor.submitProcedure(waitProcedure); LOG.info("Begin to execute " + waitProce); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(waitProce) != null); // Wait until the suspendProc executed after restart waitProcedure = (WaitProcedure) procExecutor.getProcedure(waitProce); waitProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
procExecutor.startWorkers();
@Test public void testTableProcedureDeadLockAfterRestarting() throws Exception { // let the shared procedure run first, but let it have a greater procId so when loading it will // be loaded at last. long procId1 = procExec.submitProcedure(new TableSharedProcedureWithId()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedureWithId()); procExec.startWorkers(); UTIL.waitFor(10000, () -> ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.hasQueuedThreads()); ProcedureTestingUtility.restart(procExec); ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.release(); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); }
public static void initAndStartWorkers(ProcedureExecutor<?> procExecutor, int numThreads, int numUrgentThreads, boolean abortOnCorruption, boolean startWorkers) throws IOException { procExecutor.init(numThreads, numUrgentThreads, abortOnCorruption); if (startWorkers) { procExecutor.startWorkers(); } }
@Test public void testTableProcedureSubProcedureDeadLock() throws Exception { // the shared procedure will also schedule a shared procedure, but after the exclusive procedure long procId1 = procExec.submitProcedure(new TableShardParentProcedure()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedure()); procExec.startWorkers(); UTIL.waitFor(10000, () -> procExec.getProcedures().stream().anyMatch(p -> p instanceof TableSharedProcedure)); procExec.getProcedures().stream().filter(p -> p instanceof TableSharedProcedure) .map(p -> (TableSharedProcedure) p).forEach(p -> p.latch.release()); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); } }
@Test public void testProcedureUpdatedShouldClean() throws Exception { createProcExecutor(); SuspendProcedure suspendProcedure = new SuspendProcedure(); long suspendProc = procExecutor.submitProcedure(suspendProcedure); LOG.info("Begin to execute " + suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(suspendProc) != null); // Wait until the suspendProc executed after restart suspendProcedure = (SuspendProcedure) procExecutor.getProcedure(suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
@Test public void testProcedureDeletedShouldClean() throws Exception { createProcExecutor(); WaitProcedure waitProcedure = new WaitProcedure(); long waitProce = procExecutor.submitProcedure(waitProcedure); LOG.info("Begin to execute " + waitProce); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(waitProce) != null); // Wait until the suspendProc executed after restart waitProcedure = (WaitProcedure) procExecutor.getProcedure(waitProce); waitProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
@BeforeClass public static void setUp() throws IOException { UTIL.getConfiguration().setInt("hbase.procedure.worker.stuck.threshold.msec", 6000000); procStore = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), UTIL.getDataTestDir("TestUrgentProcedureWorker")); procStore.start(1); MasterProcedureScheduler scheduler = new MasterProcedureScheduler(pid -> null); procExec = new ProcedureExecutor<>(UTIL.getConfiguration(), new TestEnv(scheduler), procStore, scheduler); procExec.init(1, 1, false); procExec.startWorkers(); }
@Test public void testTableProcedureDeadLockAfterRestarting() throws Exception { // let the shared procedure run first, but let it have a greater procId so when loading it will // be loaded at last. long procId1 = procExec.submitProcedure(new TableSharedProcedureWithId()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedureWithId()); procExec.startWorkers(); UTIL.waitFor(10000, () -> ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.hasQueuedThreads()); ProcedureTestingUtility.restart(procExec); ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.release(); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); }