public void finish() throws IOException { // notify the loader about the max proc ID loader.setMaxProcId(maxProcId); // build the procedure execution tree. When building we will verify that whether a procedure is // valid. WALProcedureTree tree = WALProcedureTree.build(procedureMap.getProcedures()); loader.load(tree.getValidProcs()); loader.handleCorrupted(tree.getCorruptedProcs()); }
public static WALProcedureTree build(Collection<ProcedureProtos.Procedure> procedures) { Map<Long, Entry> procMap = new HashMap<>(); for (ProcedureProtos.Procedure proc : procedures) { procMap.put(proc.getProcId(), new Entry(proc)); } return new WALProcedureTree(procMap); } }
private void checkReady(Entry rootEntry, Map<Long, Entry> remainingProcMap) { if (isFinished(rootEntry.proc)) { if (!rootEntry.subProcs.isEmpty()) { LOG.error("unexpected active children for root-procedure: {}", rootEntry); rootEntry.subProcs.forEach(e -> LOG.error("unexpected active children: {}", e)); addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap); } else { addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap); collectStackId(rootEntry, stackId2Proc, maxStackId); addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap); } else { addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);
private WALProcedureTree(Map<Long, Entry> procMap) { List<Entry> rootEntries = buildTree(procMap); for (Entry rootEntry : rootEntries) { checkReady(rootEntry, procMap); } checkOrphan(procMap); Comparator<ProtoAndProc> cmp = (p1, p2) -> Long.compare(p1.proto.getProcId(), p2.proto.getProcId()); Collections.sort(validProcs, cmp); Collections.sort(corruptedProcs, cmp); }
private void addAllToCorruptedAndRemoveFromProcMap(Entry entry, Map<Long, Entry> remainingProcMap) { corruptedProcs.add(new ProtoAndProc(entry.proc)); remainingProcMap.remove(entry.proc.getProcId()); for (Entry e : entry.subProcs) { addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap); } }
private void addAllToValidAndRemoveFromProcMap(Entry entry, Map<Long, Entry> remainingProcMap) { validProcs.add(new ProtoAndProc(entry.proc)); remainingProcMap.remove(entry.proc.getProcId()); for (Entry e : entry.subProcs) { addAllToValidAndRemoveFromProcMap(e, remainingProcMap); } }
private void collectStackId(Entry entry, Map<Integer, List<Entry>> stackId2Proc, MutableInt maxStackId) { if (LOG.isDebugEnabled()) { LOG.debug("Procedure {} stack ids={}", entry, entry.proc.getStackIdList()); } for (int i = 0, n = entry.proc.getStackIdCount(); i < n; i++) { int stackId = entry.proc.getStackId(i); if (stackId > maxStackId.intValue()) { maxStackId.setValue(stackId); } stackId2Proc.computeIfAbsent(stackId, k -> new ArrayList<>()).add(entry); } entry.subProcs.forEach(e -> collectStackId(e, stackId2Proc, maxStackId)); }
private WALProcedureTree(Map<Long, Entry> procMap) { List<Entry> rootEntries = buildTree(procMap); for (Entry rootEntry : rootEntries) { checkReady(rootEntry, procMap); } checkOrphan(procMap); Comparator<ProtoAndProc> cmp = (p1, p2) -> Long.compare(p1.proto.getProcId(), p2.proto.getProcId()); Collections.sort(validProcs, cmp); Collections.sort(corruptedProcs, cmp); }
private void addAllToCorruptedAndRemoveFromProcMap(Entry entry, Map<Long, Entry> remainingProcMap) { corruptedProcs.add(new ProtoAndProc(entry.proc)); remainingProcMap.remove(entry.proc.getProcId()); for (Entry e : entry.subProcs) { addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap); } }
private void addAllToValidAndRemoveFromProcMap(Entry entry, Map<Long, Entry> remainingProcMap) { validProcs.add(new ProtoAndProc(entry.proc)); remainingProcMap.remove(entry.proc.getProcId()); for (Entry e : entry.subProcs) { addAllToValidAndRemoveFromProcMap(e, remainingProcMap); } }
private void collectStackId(Entry entry, Map<Integer, List<Entry>> stackId2Proc, MutableInt maxStackId) { if (LOG.isDebugEnabled()) { LOG.debug("Procedure {} stack ids={}", entry, entry.proc.getStackIdList()); } for (int i = 0, n = entry.proc.getStackIdCount(); i < n; i++) { int stackId = entry.proc.getStackId(i); if (stackId > maxStackId.intValue()) { maxStackId.setValue(stackId); } stackId2Proc.computeIfAbsent(stackId, k -> new ArrayList<>()).add(entry); } entry.subProcs.forEach(e -> collectStackId(e, stackId2Proc, maxStackId)); }
@Test public void testDuplicatedStackId() throws IOException { TestProcedure proc0 = createProc(1, Procedure.NO_PROC_ID); proc0.addStackIndex(0); TestProcedure proc1 = createProc(2, 1); proc1.addStackIndex(1); TestProcedure proc2 = createProc(3, 2); proc2.addStackIndex(1); WALProcedureTree tree = WALProcedureTree.build(toProtos(proc0, proc1, proc2)); List<TestProcedure> validProcs = getProcs(tree.getValidProcs()); assertEquals(0, validProcs.size()); List<TestProcedure> corruptedProcs = getProcs(tree.getCorruptedProcs()); assertEquals(3, corruptedProcs.size()); assertEquals(1, corruptedProcs.get(0).getProcId()); assertEquals(2, corruptedProcs.get(1).getProcId()); assertEquals(3, corruptedProcs.get(2).getProcId()); }
private void checkReady(Entry rootEntry, Map<Long, Entry> remainingProcMap) { if (isFinished(rootEntry.proc)) { if (!rootEntry.subProcs.isEmpty()) { LOG.error("unexpected active children for root-procedure: {}", rootEntry); rootEntry.subProcs.forEach(e -> LOG.error("unexpected active children: {}", e)); addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap); } else { addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap); collectStackId(rootEntry, stackId2Proc, maxStackId); addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap); } else { addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);
public static WALProcedureTree build(Collection<ProcedureProtos.Procedure> procedures) { Map<Long, Entry> procMap = new HashMap<>(); for (ProcedureProtos.Procedure proc : procedures) { procMap.put(proc.getProcId(), new Entry(proc)); } return new WALProcedureTree(procMap); } }
@Test public void testMissingStackId() throws IOException { TestProcedure proc0 = createProc(1, Procedure.NO_PROC_ID); proc0.addStackIndex(0); TestProcedure proc1 = createProc(2, 1); proc1.addStackIndex(1); TestProcedure proc2 = createProc(3, 2); proc2.addStackIndex(3); WALProcedureTree tree = WALProcedureTree.build(toProtos(proc0, proc1, proc2)); List<TestProcedure> validProcs = getProcs(tree.getValidProcs()); assertEquals(0, validProcs.size()); List<TestProcedure> corruptedProcs = getProcs(tree.getCorruptedProcs()); assertEquals(3, corruptedProcs.size()); assertEquals(1, corruptedProcs.get(0).getProcId()); assertEquals(2, corruptedProcs.get(1).getProcId()); assertEquals(3, corruptedProcs.get(2).getProcId()); }
@Test public void testOrphan() throws IOException { TestProcedure proc0 = createProc(1, Procedure.NO_PROC_ID); proc0.addStackIndex(0); TestProcedure proc1 = createProc(2, 1); proc1.addStackIndex(1); TestProcedure proc2 = createProc(3, Procedure.NO_PROC_ID); proc2.addStackIndex(0); TestProcedure proc3 = createProc(5, 4); proc3.addStackIndex(1); WALProcedureTree tree = WALProcedureTree.build(toProtos(proc0, proc1, proc2, proc3)); List<TestProcedure> validProcs = getProcs(tree.getValidProcs()); assertEquals(3, validProcs.size()); List<TestProcedure> corruptedProcs = getProcs(tree.getCorruptedProcs()); assertEquals(1, corruptedProcs.size()); assertEquals(5, corruptedProcs.get(0).getProcId()); assertEquals(4, corruptedProcs.get(0).getParentProcId()); }
public void finish() throws IOException { // notify the loader about the max proc ID loader.setMaxProcId(maxProcId); // build the procedure execution tree. When building we will verify that whether a procedure is // valid. WALProcedureTree tree = WALProcedureTree.build(procedureMap.getProcedures()); loader.load(tree.getValidProcs()); loader.handleCorrupted(tree.getCorruptedProcs()); }
@Test public void testMissingStackId() throws IOException { TestProcedure proc0 = createProc(1, Procedure.NO_PROC_ID); proc0.addStackIndex(0); TestProcedure proc1 = createProc(2, 1); proc1.addStackIndex(1); TestProcedure proc2 = createProc(3, 2); proc2.addStackIndex(3); WALProcedureTree tree = WALProcedureTree.build(toProtos(proc0, proc1, proc2)); List<TestProcedure> validProcs = getProcs(tree.getValidProcs()); assertEquals(0, validProcs.size()); List<TestProcedure> corruptedProcs = getProcs(tree.getCorruptedProcs()); assertEquals(3, corruptedProcs.size()); assertEquals(1, corruptedProcs.get(0).getProcId()); assertEquals(2, corruptedProcs.get(1).getProcId()); assertEquals(3, corruptedProcs.get(2).getProcId()); }
@Test public void testDuplicatedStackId() throws IOException { TestProcedure proc0 = createProc(1, Procedure.NO_PROC_ID); proc0.addStackIndex(0); TestProcedure proc1 = createProc(2, 1); proc1.addStackIndex(1); TestProcedure proc2 = createProc(3, 2); proc2.addStackIndex(1); WALProcedureTree tree = WALProcedureTree.build(toProtos(proc0, proc1, proc2)); List<TestProcedure> validProcs = getProcs(tree.getValidProcs()); assertEquals(0, validProcs.size()); List<TestProcedure> corruptedProcs = getProcs(tree.getCorruptedProcs()); assertEquals(3, corruptedProcs.size()); assertEquals(1, corruptedProcs.get(0).getProcId()); assertEquals(2, corruptedProcs.get(1).getProcId()); assertEquals(3, corruptedProcs.get(2).getProcId()); }
@Test public void testOrphan() throws IOException { TestProcedure proc0 = createProc(1, Procedure.NO_PROC_ID); proc0.addStackIndex(0); TestProcedure proc1 = createProc(2, 1); proc1.addStackIndex(1); TestProcedure proc2 = createProc(3, Procedure.NO_PROC_ID); proc2.addStackIndex(0); TestProcedure proc3 = createProc(5, 4); proc3.addStackIndex(1); WALProcedureTree tree = WALProcedureTree.build(toProtos(proc0, proc1, proc2, proc3)); List<TestProcedure> validProcs = getProcs(tree.getValidProcs()); assertEquals(3, validProcs.size()); List<TestProcedure> corruptedProcs = getProcs(tree.getCorruptedProcs()); assertEquals(1, corruptedProcs.size()); assertEquals(5, corruptedProcs.get(0).getProcId()); assertEquals(4, corruptedProcs.get(0).getParentProcId()); }