public Builder withEarliestCommitRetained(Optional<HoodieInstant> earliestCommitToRetain) { this.earliestCommitToRetain = (earliestCommitToRetain.isPresent()) ? earliestCommitToRetain.get().getTimestamp() : "-1"; return this; }
@Override public Optional<byte[]> getInstantDetails(HoodieInstant instant) { return Optional.ofNullable(readCommits.get(instant.getTimestamp())); }
@Override public boolean containsOrBeforeTimelineStarts(String instant) { return instants.stream().anyMatch(s -> s.getTimestamp().equals(instant)) || isBeforeTimelineStarts(instant); }
@Override public HoodieDefaultTimeline findInstantsInRange(String startTs, String endTs) { return new HoodieDefaultTimeline(instants.stream().filter( s -> HoodieTimeline.compareTimestamps(s.getTimestamp(), startTs, GREATER) && HoodieTimeline.compareTimestamps( s.getTimestamp(), endTs, LESSER_OR_EQUAL)), details); }
@Override public HoodieDefaultTimeline findInstantsAfter(String commitTime, int numCommits) { return new HoodieDefaultTimeline( instants.stream() .filter(s -> HoodieTimeline.compareTimestamps(s.getTimestamp(), commitTime, GREATER)) .limit(numCommits), details); }
@Override public boolean isBeforeTimelineStarts(String instant) { Optional<HoodieInstant> firstCommit = firstInstant(); return firstCommit.isPresent() && HoodieTimeline.compareTimestamps(instant, firstCommit.get().getTimestamp(), LESSER); }
private Map<HeaderMetadataType, String> generateHeader(String commit) { // generate metadata Map<HeaderMetadataType, String> header = Maps.newHashMap(); header.put(HeaderMetadataType.INSTANT_TIME, metaClient.getActiveTimeline().lastInstant().get().getTimestamp()); header.put(HeaderMetadataType.TARGET_INSTANT_TIME, commit); header.put(HeaderMetadataType.COMMAND_BLOCK_TYPE, String.valueOf(HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK .ordinal())); return header; }
private Map<HeaderMetadataType, String> generateHeader(String commit) { // generate metadata Map<HeaderMetadataType, String> header = Maps.newHashMap(); header.put(HeaderMetadataType.INSTANT_TIME, metaClient.getActiveTimeline().lastInstant().get().getTimestamp()); header.put(HeaderMetadataType.TARGET_INSTANT_TIME, commit); header.put(HeaderMetadataType.COMMAND_BLOCK_TYPE, String.valueOf(HoodieCommandBlockTypeEnum.ROLLBACK_PREVIOUS_BLOCK .ordinal())); return header; }
/** * Returns the last successful write operation's instant time */ public static String latestCommit(FileSystem fs, String basePath) { HoodieTimeline timeline = allCompletedCommitsCompactions(fs, basePath); return timeline.lastInstant().get().getTimestamp(); }
/** * Returns the last successful write operation's instant time */ public static String latestCommit(FileSystem fs, String basePath) { HoodieTimeline timeline = allCompletedCommitsCompactions(fs, basePath); return timeline.lastInstant().get().getTimestamp(); }
/** * Rollback partial compactions * @param inflightInstant Inflight Compaction Instant * @param table Hoodie Table */ private void rollbackInflightCompaction(HoodieInstant inflightInstant, HoodieTable table) throws IOException { table.rollback(jsc, ImmutableList.copyOf(new String[] { inflightInstant.getTimestamp() })); // Revert instant state file table.getActiveTimeline().revertCompactionInflightToRequested(inflightInstant); }
@CliCommand(value = "savepoints show", help = "Show the savepoints") public String showSavepoints() throws IOException { HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline(); HoodieTimeline timeline = activeTimeline.getSavePointTimeline().filterCompletedInstants(); List<HoodieInstant> commits = timeline.getInstants().collect(Collectors.toList()); String[][] rows = new String[commits.size()][]; Collections.reverse(commits); for (int i = 0; i < commits.size(); i++) { HoodieInstant commit = commits.get(i); rows[i] = new String[] {commit.getTimestamp()}; } return HoodiePrintHelper.print(new String[] {"SavepointTime"}, rows); }
/** * Rollback partial compactions * @param inflightInstant Inflight Compaction Instant * @param table Hoodie Table */ private void rollbackInflightCompaction(HoodieInstant inflightInstant, HoodieTable table) throws IOException { table.rollback(jsc, ImmutableList.copyOf(new String[] { inflightInstant.getTimestamp() })); // Revert instant state file table.getActiveTimeline().revertCompactionInflightToRequested(inflightInstant); }
/** * A FileSlice is considered committed, if one of the following is true - There is a committed * data file - There are some log files, that are based off a commit or delta commit */ private boolean isFileSliceCommitted(FileSlice slice) { String maxCommitTime = lastInstant.get().getTimestamp(); return timeline.containsOrBeforeTimelineStarts(slice.getBaseInstantTime()) && HoodieTimeline.compareTimestamps(slice.getBaseInstantTime(), maxCommitTime, HoodieTimeline.LESSER_OR_EQUAL); }
/** * Return all pending compactions with instant time for clients to decide what to compact next. * @return */ public List<Pair<String, HoodieCompactionPlan>> getPendingCompactions() { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), hoodieTable.getMetaClient().getBasePath(), true); return CompactionUtils.getAllPendingCompactionPlans(metaClient).stream() .map(instantWorkloadPair -> Pair.of(instantWorkloadPair.getKey().getTimestamp(), instantWorkloadPair.getValue())) .collect(Collectors.toList()); } }
void updateLastCommitTimeSynced() { // Set the last commit time from the TBLproperties String lastCommitSynced = activeTimeline.lastInstant().get().getTimestamp(); try { Table table = client.getTable(syncConfig.databaseName, syncConfig.tableName); table.putToParameters(HOODIE_LAST_COMMIT_TIME_SYNC, lastCommitSynced); client.alter_table(syncConfig.databaseName, syncConfig.tableName, table, true); } catch (Exception e) { throw new HoodieHiveSyncException( "Failed to get update last commit time synced to " + lastCommitSynced, e); } }
private boolean checkIfValidCommit(HoodieTableMetaClient metaClient, String commitTs) { HoodieTimeline commitTimeline = metaClient.getActiveTimeline().filterCompletedInstants(); // Check if the last commit ts for this row is 1) present in the timeline or // 2) is less than the first commit ts in the timeline return !commitTimeline.empty() && (commitTimeline .containsInstant(new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTs)) || HoodieTimeline .compareTimestamps(commitTimeline.firstInstant().get().getTimestamp(), commitTs, HoodieTimeline.GREATER)); }
private boolean checkIfValidCommit(HoodieTableMetaClient metaClient, String commitTs) { HoodieTimeline commitTimeline = metaClient.getActiveTimeline().filterCompletedInstants(); // Check if the last commit ts for this row is 1) present in the timeline or // 2) is less than the first commit ts in the timeline return !commitTimeline.empty() && (commitTimeline .containsInstant(new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, commitTs)) || HoodieTimeline .compareTimestamps(commitTimeline.firstInstant().get().getTimestamp(), commitTs, HoodieTimeline.GREATER)); }
private void scheduleCompaction(String compactionInstantTime, HoodieWriteClient client, HoodieWriteConfig cfg) throws IOException { client.scheduleCompactionAtInstant(compactionInstantTime, Optional.empty()); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); HoodieInstant instant = metaClient.getActiveTimeline().filterPendingCompactionTimeline().lastInstant().get(); assertEquals("Last compaction instant must be the one set", instant.getTimestamp(), compactionInstantTime); }
private void moveCompactionFromRequestedToInflight(String compactionInstantTime, HoodieWriteClient client, HoodieWriteConfig cfg) throws IOException { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); HoodieInstant compactionInstant = HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime); HoodieCompactionPlan workload = AvroUtils.deserializeCompactionPlan( metaClient.getActiveTimeline().getInstantAuxiliaryDetails(compactionInstant).get()); metaClient.getActiveTimeline().transitionCompactionRequestedToInflight(compactionInstant); HoodieInstant instant = metaClient.getActiveTimeline().reload().filterPendingCompactionTimeline().getInstants() .filter(in -> in.getTimestamp().equals(compactionInstantTime)).findAny().get(); assertTrue("Instant must be marked inflight", instant.isInflight()); }