/** * Parses a directory of WALs building up ProcedureState. * For testing parse and profiling. * @param args Include pointer to directory of WAL files for a store instance to parse & load. */ public static void main(String [] args) throws IOException { Configuration conf = HBaseConfiguration.create(); if (args == null || args.length != 1) { System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR."); System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR"); System.exit(-1); } WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null, new WALProcedureStore.LeaseRecovery() { @Override public void recoverFileLease(FileSystem fs, Path path) throws IOException { // no-op } }); try { store.start(16); ProcedureExecutor<?> pe = new ProcedureExecutor<>(conf, new Object()/*Pass anything*/, store); pe.init(1, true); } finally { store.stop(true); } } }
public void setUpProcedureStore() throws IOException { Path testDir = UTIL.getDataTestDir(); FileSystem fs = testDir.getFileSystem(conf); Path logDir = new Path(testDir, "proc-logs"); System.out.println("\n\nLogs directory : " + logDir.toString() + "\n\n"); fs.delete(logDir, true); store = ProcedureTestingUtility.createWalStore(conf, logDir); store.start(1); store.recoverLease(); store.load(new LoadCounter()); }
private void createProcedureExecutor() throws IOException { MasterProcedureEnv procEnv = new MasterProcedureEnv(this); procedureStore = new WALProcedureStore(conf, new MasterProcedureEnv.WALStoreLeaseRecovery(this)); procedureStore.registerListener(new ProcedureStoreListener() { @Override public void abortProcess() { abort("The Procedure Store lost the lease", null); } }); MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler(); procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler); configurationManager.registerObserver(procEnv); int cpus = Runtime.getRuntime().availableProcessors(); final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max( (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); final boolean abortOnCorruption = conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); procedureStore.start(numThreads); // Just initialize it but do not start the workers, we will start the workers later by calling // startProcedureExecutor. See the javadoc for finishActiveMasterInitialization for more // details. procedureExecutor.init(numThreads, abortOnCorruption); procEnv.getRemoteDispatcher().start(); }
private void storeRestart(ProcedureStore.ProcedureLoader loader) throws IOException { System.out.println("Restarting procedure store to read back the WALs"); store.stop(false); store.start(1); store.recoverLease(); long startTime = currentTimeMillis(); store.load(loader); long timeTaken = System.currentTimeMillis() - startTime; System.out.println("******************************************"); System.out.println("Load time : " + (timeTaken / 1000.0f) + "sec"); System.out.println("******************************************"); System.out.println("Raw format for scripts"); System.out.println(String.format("RESULT [%s=%s, %s=%s, %s=%s, %s=%s, %s=%s, " + "total_time_ms=%s]", NUM_PROCS_OPTION.getOpt(), numProcs, STATE_SIZE_OPTION.getOpt(), serializedState.length, UPDATES_PER_PROC_OPTION.getOpt(), updatesPerProc, DELETE_PROCS_FRACTION_OPTION.getOpt(), deleteProcsFraction, NUM_WALS_OPTION.getOpt(), numWals, timeTaken)); }
private void setupProcedureStore() throws IOException { Path testDir = UTIL.getDataTestDir(); FileSystem fs = testDir.getFileSystem(conf); Path logDir = new Path(testDir, "proc-logs"); System.out.println("Logs directory : " + logDir.toString()); fs.delete(logDir, true); if ("nosync".equals(syncType)) { store = new NoSyncWalProcedureStore(conf, logDir); } else { store = ProcedureTestingUtility.createWalStore(conf, logDir); } store.start(numThreads); store.recoverLease(); store.load(new ProcedureTestingUtility.LoadCounter()); System.out.println("Starting new log : " + store.getActiveLogs().get(store.getActiveLogs().size() - 1)); }
private void setupDFS() throws Exception { Configuration conf = UTIL.getConfiguration(); MiniDFSCluster dfs = UTIL.startMiniDFSCluster(3); CommonFSUtils.setWALRootDir(conf, new Path(conf.get("fs.defaultFS"), "/tmp/wal")); Path logDir = new Path(new Path(dfs.getFileSystem().getUri()), "/test-logs"); store = ProcedureTestingUtility.createWalStore(conf, logDir); store.registerListener(stopProcedureListener); store.start(8); store.recoverLease(); }
private void createProcExecutor() throws Exception { logDir = new Path(testDir, name.getMethodName()); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), null, procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true, true); }
private void createStoreAndExecutor() throws IOException { UTIL.getConfiguration().setInt(CompletedProcedureCleaner.CLEANER_INTERVAL_CONF_KEY, 1000); Path logDir = UTIL.getDataTestDir(name.getMethodName()); STORE = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), logDir); STORE.start(1); EXEC = new ProcedureExecutor<Void>(UTIL.getConfiguration(), null, STORE); ProcedureTestingUtility.initAndStartWorkers(EXEC, 1, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); setupConfig(htu.getConfiguration()); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procStore.start(PROCEDURE_STORE_SLOTS); procStore.recoverLease(); procStore.load(new LoadCounter()); }
procStore2.start(1); procStore2.recoverLease();
@Before public void setUp() throws IOException { procStore = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), UTIL.getDataTestDir(name.getMethodName())); procStore.start(2); procExec = new ProcedureExecutor<Void>(UTIL.getConfiguration(), null, procStore); ProcedureTestingUtility.initAndStartWorkers(procExec, 2, true); }
@Before public void setUp() throws IOException { UTIL.getConfiguration().setInt("hbase.procedure.worker.stuck.threshold.msec", 6000000); procStore = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), UTIL.getDataTestDir(name.getMethodName())); procStore.start(1); MasterProcedureScheduler scheduler = new MasterProcedureScheduler(pid -> null); procExec = new ProcedureExecutor<>(UTIL.getConfiguration(), new TestEnv(scheduler), procStore, scheduler); procExec.init(1, false); }
backupStore3.start(1); backupStore3.recoverLease();
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); setupConfiguration(htu.getConfiguration()); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procStore.start(PROCEDURE_STORE_SLOTS); procStore.recoverLease(); LoadCounter loader = new LoadCounter(); procStore.load(loader); assertEquals(0, loader.getMaxProcId()); assertEquals(0, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }
procStore.start(PROCEDURE_STORE_SLOTS); procStore.recoverLease(); procStore.load(loader);
private void setupDFS() throws Exception { MiniDFSCluster dfs = UTIL.startMiniDFSCluster(3); Path logDir = new Path(new Path(dfs.getFileSystem().getUri()), "/test-logs"); store = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), logDir); store.registerListener(stopProcedureListener); store.start(8); store.recoverLease(); }
private void createProcExecutor() throws Exception { logDir = new Path(testDir, name.getMethodName()); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), null, procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true, true); }
@Before public void setUp() throws IOException { procStore = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), UTIL.getDataTestDir(name.getMethodName())); procStore.start(2); procExec = new ProcedureExecutor<Void>(UTIL.getConfiguration(), null, procStore); ProcedureTestingUtility.initAndStartWorkers(procExec, 2, true); }
@BeforeClass public static void setUp() throws IOException { UTIL.getConfiguration().setInt("hbase.procedure.worker.stuck.threshold.msec", 6000000); procStore = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), UTIL.getDataTestDir("TestUrgentProcedureWorker")); procStore.start(1); MasterProcedureScheduler scheduler = new MasterProcedureScheduler(pid -> null); procExec = new ProcedureExecutor<>(UTIL.getConfiguration(), new TestEnv(scheduler), procStore, scheduler); procExec.init(1, 1, false); procExec.startWorkers(); }
@Before public void setUp() throws IOException { UTIL.getConfiguration().setInt("hbase.procedure.worker.stuck.threshold.msec", 6000000); procStore = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), UTIL.getDataTestDir(name.getMethodName())); procStore.start(1); MasterProcedureScheduler scheduler = new MasterProcedureScheduler(pid -> null); procExec = new ProcedureExecutor<>(UTIL.getConfiguration(), new TestEnv(scheduler), procStore, scheduler); procExec.init(1, false); }