/** * Performs Compaction for the workload stored in instant-time * @param compactionInstantTime Compaction Instant Time * @return * @throws IOException */ public JavaRDD<WriteStatus> compact(String compactionInstantTime) throws IOException { return compact(compactionInstantTime, config.shouldAutoCommit()); }
/** * Performs Compaction for the workload stored in instant-time * @param compactionInstantTime Compaction Instant Time * @return * @throws IOException */ public JavaRDD<WriteStatus> compact(String compactionInstantTime) throws IOException { return compact(compactionInstantTime, config.shouldAutoCommit()); }
private void commitOnAutoCommit(String commitTime, JavaRDD<WriteStatus> resultRDD, String actionType) { if (config.shouldAutoCommit()) { logger.info("Auto commit enabled: Committing " + commitTime); boolean commitResult = commit(commitTime, resultRDD, Optional.empty(), actionType); if (!commitResult) { throw new HoodieCommitException("Failed to commit " + commitTime); } } else { logger.info("Auto commit disabled for " + commitTime); } }
private void commitOnAutoCommit(String commitTime, JavaRDD<WriteStatus> resultRDD, String actionType) { if (config.shouldAutoCommit()) { logger.info("Auto commit enabled: Committing " + commitTime); boolean commitResult = commit(commitTime, resultRDD, Optional.empty(), actionType); if (!commitResult) { throw new HoodieCommitException("Failed to commit " + commitTime); } } else { logger.info("Auto commit disabled for " + commitTime); } }
private List<WriteStatus> createNextDeltaCommit(String instantTime, List<HoodieRecord> records, HoodieWriteClient client, HoodieTableMetaClient metaClient, HoodieWriteConfig cfg, boolean skipCommit) { JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1); client.startCommitWithTime(instantTime); JavaRDD<WriteStatus> statuses = client.upsert(writeRecords, instantTime); List<WriteStatus> statusList = statuses.collect(); TestHoodieClientBase.assertNoWriteErrors(statusList); if (!cfg.shouldAutoCommit() && !skipCommit) { client.commit(instantTime, statuses); } Optional<HoodieInstant> deltaCommit = metaClient.getActiveTimeline().reload().getDeltaCommitTimeline() .filterCompletedInstants().lastInstant(); if (skipCommit && !cfg.shouldAutoCommit()) { assertTrue("Delta commit should not be latest instant", deltaCommit.get().getTimestamp().compareTo(instantTime) < 0); } else { assertTrue(deltaCommit.isPresent()); assertEquals("Delta commit should be latest instant", instantTime, deltaCommit.get().getTimestamp()); } return statusList; }
List<WriteStatus> statusList = statuses.collect(); if (!cfg.shouldAutoCommit()) { client.commit(firstInstant, statuses);