private Path archivePath(Path p) { Path retPath = null; Path tmp = p; for (int i=0; i< p.depth(); i++) { if (tmp.toString().endsWith(".har")) { retPath = tmp; break; } tmp = tmp.getParent(); } return retPath; }
private static boolean isPathWithinSubtree(Path path, Path subtree, int subtreeDepth) { while(path != null){ if (subtreeDepth > path.depth()) { return false; } if(subtree.equals(path)){ return true; } path = path.getParent(); } return false; }
/** * Checks whenever path is inside the given subtree * * return true iff * * path = subtree * * subtreeContains(path,d) for any descendant of the subtree node * @param path the path in question * @param subtree * * @return */ public static boolean isPathWithinSubtree(Path path, Path subtree) { return isPathWithinSubtree(path, subtree, subtree.depth()); }
private Path makeRelative(String initial, Path p) { String scheme = this.uri.getScheme(); String authority = this.uri.getAuthority(); Path root = new Path(Path.SEPARATOR); if (root.compareTo(p) == 0) return new Path(scheme, authority, initial); Path retPath = new Path(p.getName()); Path parent = p.getParent(); for (int i=0; i < p.depth()-1; i++) { retPath = new Path(parent.getName(), retPath); parent = parent.getParent(); } return new Path(new Path(scheme, authority, initial), retPath.toString()); }
/** * Remove an element (i.e. either a {@link DataNode} or a {@link FlowEdge} from the {@link FlowGraph} for * a renamed or deleted {@link DataNode} or {@link FlowEdge} file. * @param change */ @Override public void removeChange(DiffEntry change) { Path path = new Path(change.getOldPath()); if (path.depth() == NODE_FILE_DEPTH) { removeDataNode(change); } else if (path.depth() == EDGE_FILE_DEPTH) { removeFlowEdge(change); } }
/** * Add an element (i.e., a {@link DataNode}, or a {@link FlowEdge} to * the {@link FlowGraph} for an added, updated or modified node or edge file. * @param change */ @Override public void addChange(DiffEntry change) { Path path = new Path(change.getNewPath()); if (path.depth() == NODE_FILE_DEPTH) { addDataNode(change); } else if (path.depth() == EDGE_FILE_DEPTH) { addFlowEdge(change); } }
/** * check whether the file has the proper naming and hierarchy * @param file the relative path from the repo root * @return false if the file does not conform */ private boolean checkFilePath(String file, int depth) { // The file is either a node file or an edge file and needs to be stored at either: // flowGraphDir/nodeName/nodeName.properties (if it is a node file), or // flowGraphDir/nodeName/nodeName/edgeName.properties (if it is an edge file) Path filePath = new Path(file); String fileExtension = Files.getFileExtension(filePath.getName()); if (filePath.depth() != depth || !checkFileLevelRelativeToRoot(filePath, depth) || !(this.javaPropsExtensions.contains(fileExtension))) { log.warn("Changed file does not conform to directory structure and file name format, skipping: " + filePath); return false; } return true; }
@BeforeClass public static void setUp() throws Exception { htu = new HBaseCommonTestingUtility(); htu.getConfiguration().setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, true); // NOTE: The executor will be created by each test testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), null, procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); htu.getConfiguration().setInt(WALProcedureStore.SYNC_WAIT_MSEC_CONF_KEY, 25); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procEnv = new TestProcedureEnv(); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), procEnv, procStore); procStore.start(NUM_THREADS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, 1, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), new ProcEnv(), procStore); procStore.start(1); ProcedureTestingUtility.initAndStartWorkers(procExecutor, 1, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); setupConfig(htu.getConfiguration()); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procStore.start(PROCEDURE_STORE_SLOTS); procStore.recoverLease(); procStore.load(new LoadCounter()); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procEnv = new TestProcEnv(); procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), procEnv, procStore); procExecutor.testing = new ProcedureExecutor.Testing(); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procEnv = new TestProcEnv(); procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<TestProcEnv>(htu.getConfiguration(), procEnv, procStore); procExecutor.testing = new ProcedureExecutor.Testing(); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); }
@BeforeClass public static void setUp() throws Exception { htu = new HBaseCommonTestingUtility(); // NOTE: The executor will be created by each test procEnv = new TestProcEnv(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), procEnv, procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility .initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); Path testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procEnv = new TestProcEnv(); procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), procEnv, procStore); procExecutor.testing = new ProcedureExecutor.Testing(); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procEnv = new TestProcEnv(); procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), procEnv, procStore); procExecutor.testing = new ProcedureExecutor.Testing(); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); procSleepInterval = 0; }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procRunnables = new TestScheduler(); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), new TestProcEnv(), procStore, procRunnables); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); setupConfiguration(htu.getConfiguration()); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); assertTrue(testDir.depth() > 1); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procStore.start(PROCEDURE_STORE_SLOTS); procStore.recoverLease(); LoadCounter loader = new LoadCounter(); procStore.load(loader); assertEquals(0, loader.getMaxProcId()); assertEquals(0, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }