@Test public void testGetAllPendingCompactionOperationsForEmptyCompactions() throws IOException { // Case where there are 4 compaction requests and all are empty. setupAndValidateCompactionOperations(metaClient, false, 0, 0, 0, 0); }
int numEntriesInPlan1, int numEntriesInPlan2, int numEntriesInPlan3, int numEntriesInPlan4) throws IOException { HoodieCompactionPlan plan1 = createCompactionPlan(metaClient, "000", "001", numEntriesInPlan1, true, true); HoodieCompactionPlan plan2 = createCompactionPlan(metaClient, "002", "003", numEntriesInPlan2, false, true); HoodieCompactionPlan plan3 = createCompactionPlan(metaClient, "004", "005", numEntriesInPlan3, true, false); HoodieCompactionPlan plan4 = createCompactionPlan(metaClient, "006", "007", numEntriesInPlan4, false, false); scheduleInflightCompaction(metaClient, "001", plan1); scheduleInflightCompaction(metaClient, "003", plan2); scheduleInflightCompaction(metaClient, "005", plan3); scheduleInflightCompaction(metaClient, "007", plan4); } else { scheduleCompaction(metaClient, "001", plan1); scheduleCompaction(metaClient, "003", plan2); scheduleCompaction(metaClient, "005", plan3); scheduleCompaction(metaClient, "007", plan4); createDeltaCommit(metaClient, "000"); createDeltaCommit(metaClient, "002"); createDeltaCommit(metaClient, "004"); createDeltaCommit(metaClient, "006"); generateExpectedCompactionOperations(Arrays.asList(plan1, plan2, plan3, plan4), baseInstantsToCompaction);
@Test(expected = IllegalStateException.class) public void testGetAllPendingCompactionOperationsWithDupFileId() throws IOException { // Case where there is duplicate fileIds in compaction requests HoodieCompactionPlan plan1 = createCompactionPlan(metaClient, "000", "001", 10, true, true); HoodieCompactionPlan plan2 = createCompactionPlan(metaClient, "002", "003", 0, false, false); scheduleCompaction(metaClient, "001", plan1); scheduleCompaction(metaClient, "003", plan2); // schedule same plan again so that there will be duplicates scheduleCompaction(metaClient, "005", plan1); metaClient = new HoodieTableMetaClient(metaClient.getHadoopConf(), basePath, true); Map<String, Pair<String, HoodieCompactionOperation>> res = CompactionUtils.getAllPendingCompactionOperations(metaClient); }
public static void scheduleInflightCompaction(HoodieTableMetaClient metaClient, String instantTime, HoodieCompactionPlan compactionPlan) throws IOException { scheduleCompaction(metaClient, instantTime, compactionPlan); metaClient.getActiveTimeline().transitionCompactionRequestedToInflight( new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, instantTime)); }
@Test public void testGetAllPendingCompactionOperations() throws IOException { // Case where there are 4 compaction requests where 1 is empty. setupAndValidateCompactionOperations(metaClient, false, 10, 10, 10, 0); }
@Test public void testGetAllPendingInflightCompactionOperations() throws IOException { // Case where there are 4 compaction requests where 1 is empty. All of them are marked inflight setupAndValidateCompactionOperations(metaClient, true, 10, 10, 10, 0); }
@Test public void testUnscheduleCompactionFileId() throws Exception { int numEntriesPerInstant = 10; CompactionTestUtils .setupAndValidateCompactionOperations(metaClient, false, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant); Map<String, CompactionOperation> instantsWithOp = Arrays.asList("001", "003", "005", "007").stream().map(instant -> { try { return Pair.of(instant, CompactionUtils.getCompactionPlan(metaClient, instant)); } catch (IOException ioe) { throw new HoodieException(ioe); } }).map(instantWithPlan -> instantWithPlan.getRight().getOperations().stream().map(op -> Pair.of( instantWithPlan.getLeft(), CompactionOperation.convertFromAvroRecordInstance(op))).findFirst().get()) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); // THere are delta-commits after compaction instant validateUnScheduleFileId(client, "000", "001", instantsWithOp.get("001"), 2); // THere are delta-commits after compaction instant validateUnScheduleFileId(client, "002", "003", instantsWithOp.get("003"), 2); // THere are no delta-commits after compaction instant validateUnScheduleFileId(client, "004", "005", instantsWithOp.get("005"), 0); // THere are no delta-commits after compaction instant validateUnScheduleFileId(client, "006", "007", instantsWithOp.get("007"), 0); }
@Test public void testRepairCompactionPlan() throws Exception { int numEntriesPerInstant = 10; CompactionTestUtils .setupAndValidateCompactionOperations(metaClient,false, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant); // THere are delta-commits after compaction instant validateRepair("000", "001", numEntriesPerInstant, 2 * numEntriesPerInstant); // THere are delta-commits after compaction instant validateRepair("002", "003", numEntriesPerInstant, 2 * numEntriesPerInstant); // THere are no delta-commits after compaction instant validateRepair("004", "005", numEntriesPerInstant, 0); // THere are no delta-commits after compaction instant validateRepair("006", "007", numEntriesPerInstant, 0); }
@Test public void testUnscheduleCompactionPlan() throws Exception { int numEntriesPerInstant = 10; CompactionTestUtils .setupAndValidateCompactionOperations(metaClient, false, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant, numEntriesPerInstant); // THere are delta-commits after compaction instant validateUnSchedulePlan(client, "000", "001", numEntriesPerInstant, 2 * numEntriesPerInstant); // THere are delta-commits after compaction instant validateUnSchedulePlan(client, "002", "003", numEntriesPerInstant, 2 * numEntriesPerInstant); // THere are no delta-commits after compaction instant validateUnSchedulePlan(client, "004", "005", numEntriesPerInstant, 0); // THere are no delta-commits after compaction instant validateUnSchedulePlan(client, "006", "007", numEntriesPerInstant, 0); }