private void visit(SparkTran child, Set<SparkTran> seen, List<SparkTran> result) { if (seen.contains(child)) { // don't visit multiple times return; } seen.add(child); for (SparkTran parent: getParents(child)) { if (!seen.contains(parent)) { visit(parent, seen, result); } } result.add(child); }
private void collectLeafTrans(SparkTran leaf, List<SparkTran> reduceTrans) { List<SparkTran> parents = getParents(leaf); if (parents.size() > 0) { SparkTran nextLeaf = null; for (SparkTran leafTran : parents) { if (leafTran instanceof ReduceTran) { reduceTrans.add(leafTran); } else { if (getParents(leafTran).size() > 0) nextLeaf = leafTran; } } if (nextLeaf != null) collectLeafTrans(nextLeaf, reduceTrans); } }
private void visit(SparkTran child, Set<SparkTran> seen, List<SparkTran> result) { if (seen.contains(child)) { // don't visit multiple times return; } seen.add(child); for (SparkTran parent: getParents(child)) { if (!seen.contains(parent)) { visit(parent, seen, result); } } result.add(child); }
private void getSparkPlan(SparkTran tran, StringBuilder sparkPlan) { List<SparkTran> parents = getParents(tran); List<SparkTran> nextLeaf = new ArrayList<SparkTran>(); if (parents.size() > 0) { if (getParents(leaf).size() > 0 && !(leaf instanceof ReduceTran)) { nextLeaf.add(leaf);
private void logLeafTran(List<SparkTran> parent, StringBuilder sparkPlan) { sparkPlan.append(" <-- "); boolean isFirst = true; for (SparkTran sparkTran : parent) { List<SparkTran> parents = getParents(sparkTran); SparkTran leaf = parents.get(0); if (isFirst) { sparkPlan.append("( " + leaf.getName()); if (leaf instanceof ShuffleTran) { logShuffleTranStatus((ShuffleTran) leaf, sparkPlan); } else { logCacheStatus(leaf, sparkPlan); } isFirst = false; } else { sparkPlan.append("," + leaf.getName()); if (leaf instanceof ShuffleTran) { logShuffleTranStatus((ShuffleTran) leaf, sparkPlan); } else { logCacheStatus(leaf, sparkPlan); } } } sparkPlan.append(" ) "); }
for (SparkTran tran : getAllTrans()) { JavaPairRDD<HiveKey, BytesWritable> rdd = null; List<SparkTran> parents = getParents(tran); if (parents.size() == 0) {
for (SparkTran tran : getAllTrans()) { JavaPairRDD<HiveKey, BytesWritable> rdd = null; List<SparkTran> parents = getParents(tran); if (parents.size() == 0) {
private void visit(SparkTran child, Set<SparkTran> seen, List<SparkTran> result) { if (seen.contains(child)) { // don't visit multiple times return; } seen.add(child); for (SparkTran parent: getParents(child)) { if (!seen.contains(parent)) { visit(parent, seen, result); } } result.add(child); }
for (SparkTran tran : getAllTrans()) { JavaPairRDD<HiveKey, BytesWritable> rdd = null; List<SparkTran> parents = getParents(tran); if (parents.size() == 0) {