public static int[] calculateAllLevelCount(CubeDesc cube) { int levels = cube.getRowkey().getNCuboidBuildLevels(); int[] allLevelCounts = new int[levels + 1]; CuboidScheduler scheduler = new CuboidScheduler(cube); LinkedList<Long> nextQueue = new LinkedList<Long>(); LinkedList<Long> currentQueue = new LinkedList<Long>(); long baseCuboid = Cuboid.getBaseCuboidId(cube); currentQueue.push(baseCuboid); for (int i = 0; i <= levels; i++) { allLevelCounts[i] = currentQueue.size(); while (!currentQueue.isEmpty()) { long cuboid = currentQueue.pop(); Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid); nextQueue.addAll(spnanningCuboids); } currentQueue = nextQueue; nextQueue = new LinkedList<Long>(); } return allLevelCounts; }
Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result) { final int groupRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getNCuboidBuildLevels(); final int totalRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getRowKeyColumns().length; final String jobId = result.getId(); final CubeJoinedFlatTableDesc intermediateTableDesc = new CubeJoinedFlatTableDesc(seg.getCubeDesc(), seg); final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId); final String intermediateHiveTableLocation = getIntermediateHiveTableLocation(intermediateTableDesc, jobId); final String factDistinctColumnsPath = getFactDistinctColumnsPath(seg, jobId); final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId); result.addTask(intermediateHiveTableStep); result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId)); result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath)); // base cuboid step final MapReduceExecutable baseCuboidStep = createBaseCuboidStep(seg, intermediateHiveTableLocation, cuboidOutputTempPath); result.addTask(baseCuboidStep); // n dim cuboid steps for (int i = 1; i <= groupRowkeyColumnsCount; i++) { int dimNum = totalRowkeyColumnsCount - i; result.addTask(createNDimensionCuboidStep(seg, cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount)); } return new Pair<AbstractExecutable, AbstractExecutable>(intermediateHiveTableStep, baseCuboidStep); }
public static int[] calculateAllLevelCount(CubeDesc cube) { int levels = cube.getRowkey().getNCuboidBuildLevels(); int[] allLevelCounts = new int[levels + 1]; CuboidScheduler scheduler = new CuboidScheduler(cube); LinkedList<Long> nextQueue = new LinkedList<Long>(); LinkedList<Long> currentQueue = new LinkedList<Long>(); long baseCuboid = Cuboid.getBaseCuboidId(cube); currentQueue.push(baseCuboid); for (int i = 0; i <= levels; i++) { allLevelCounts[i] = currentQueue.size(); while (!currentQueue.isEmpty()) { long cuboid = currentQueue.pop(); Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid); nextQueue.addAll(spnanningCuboids); } currentQueue = nextQueue; nextQueue = new LinkedList<Long>(); } return allLevelCounts; }
Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result, List<String> toDeletePaths) { final int groupRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getNCuboidBuildLevels(); final int totalRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getRowKeyColumns().length; final String jobId = result.getId(); final CubeJoinedFlatTableDesc intermediateTableDesc = new CubeJoinedFlatTableDesc(seg.getCubeDesc(), seg); final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId); final String intermediateHiveTableLocation = getIntermediateHiveTableLocation(intermediateTableDesc, jobId); final String factDistinctColumnsPath = getFactDistinctColumnsPath(seg, jobId); final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId); result.addTask(intermediateHiveTableStep); result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId)); result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath)); // base cuboid step final MapReduceExecutable baseCuboidStep = createBaseCuboidStep(seg, intermediateHiveTableLocation, cuboidOutputTempPath); result.addTask(baseCuboidStep); // n dim cuboid steps for (int i = 1; i <= groupRowkeyColumnsCount; i++) { int dimNum = totalRowkeyColumnsCount - i; result.addTask(createNDimensionCuboidStep(seg, cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount)); } toDeletePaths.add(intermediateHiveTableLocation); toDeletePaths.add(factDistinctColumnsPath); return new Pair<AbstractExecutable, AbstractExecutable>(intermediateHiveTableStep, baseCuboidStep); }