protected Stream<HoodieInstant> filterInstantsByAction(String action) { return instants.stream().filter(s -> s.getAction().equals(action)); }
/** * Get a timeline of a specific set of actions. useful to create a merged timeline of multiple * actions * * @param actions actions allowed in the timeline */ public HoodieTimeline getTimelineOfActions(Set<String> actions) { return new HoodieDefaultTimeline(instants.stream().filter(s -> actions.contains(s.getAction())), (Function<HoodieInstant, Optional<byte[]>> & Serializable) this::getInstantDetails); }
@Override public HoodieTimeline filterPendingCompactionTimeline() { return new HoodieDefaultTimeline( instants.stream().filter(s -> s.getAction().equals(HoodieTimeline.COMPACTION_ACTION)), details); }
@Override public HoodieTimeline filterCompletedAndCompactionInstants() { return new HoodieDefaultTimeline(instants.stream().filter(s -> { return !s.isInflight() || s.getAction().equals(HoodieTimeline.COMPACTION_ACTION); }), details); }
@Override public HoodieTimeline filterInflightsExcludingCompaction() { return new HoodieDefaultTimeline(instants.stream().filter(instant -> { return instant.isInflight() && (!instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)); }), details); }
public void deleteCompactionRequested(HoodieInstant instant) { Preconditions.checkArgument(instant.isRequested()); Preconditions.checkArgument(instant.getAction() == HoodieTimeline.COMPACTION_ACTION); deleteInstantFile(instant); }
public void saveToCompactionRequested(HoodieInstant instant, Optional<byte[]> content) { Preconditions.checkArgument(instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)); // Write workload to auxiliary folder createFileInAuxiliaryFolder(instant, content); createFileInMetaPath(instant.getFileName(), content); }
/** * Transition Compaction State from requested to inflight * * @param requestedInstant Requested instant * @return inflight instant */ public HoodieInstant transitionCompactionRequestedToInflight(HoodieInstant requestedInstant) { Preconditions.checkArgument(requestedInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)); Preconditions.checkArgument(requestedInstant.isRequested()); HoodieInstant inflightInstant = new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, requestedInstant.getTimestamp()); transitionState(requestedInstant, inflightInstant, Optional.empty()); return inflightInstant; }
static HoodieInstant getCompletedInstant(final HoodieInstant instant) { return new HoodieInstant(false, instant.getAction(), instant.getTimestamp()); }
static HoodieInstant getInflightInstant(final HoodieInstant instant) { return new HoodieInstant(true, instant.getAction(), instant.getTimestamp()); }
/** * Revert compaction State from inflight to requested * * @param inflightInstant Inflight Instant * @return requested instant */ public HoodieInstant revertCompactionInflightToRequested(HoodieInstant inflightInstant) { Preconditions.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)); Preconditions.checkArgument(inflightInstant.isInflight()); HoodieInstant requestedInstant = new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, inflightInstant.getTimestamp()); transitionState(inflightInstant, requestedInstant, Optional.empty()); return requestedInstant; }
/** * Transition Compaction State from inflight to Committed * * @param inflightInstant Inflight instant * @param data Extra Metadata * @return commit instant */ public HoodieInstant transitionCompactionInflightToComplete(HoodieInstant inflightInstant, Optional<byte[]> data) { Preconditions.checkArgument(inflightInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)); Preconditions.checkArgument(inflightInstant.isInflight()); HoodieInstant commitInstant = new HoodieInstant(State.COMPLETED, COMMIT_ACTION, inflightInstant.getTimestamp()); transitionState(inflightInstant, commitInstant, data); return commitInstant; }
private boolean deleteArchivedInstants(List<HoodieInstant> archivedInstants) throws IOException { log.info("Deleting instants " + archivedInstants); boolean success = true; for (HoodieInstant archivedInstant : archivedInstants) { Path commitFile = new Path(metaClient.getMetaPath(), archivedInstant.getFileName()); try { if (metaClient.getFs().exists(commitFile)) { success &= metaClient.getFs().delete(commitFile, false); log.info("Archived and deleted instant file " + commitFile); } } catch (IOException e) { throw new HoodieIOException("Failed to delete archived instant " + archivedInstant, e); } } // Remove older meta-data from auxiliary path too Optional<HoodieInstant> latestCommitted = archivedInstants.stream() .filter(i -> { return i.isCompleted() && (i.getAction().equals(COMMIT_ACTION) || (i.getAction().equals(DELTA_COMMIT_ACTION))); }) .sorted(Comparator.comparing(HoodieInstant::getTimestamp).reversed()).findFirst(); if (latestCommitted.isPresent()) { success &= deleteAllInstantsOlderorEqualsInAuxMetaFolder(latestCommitted.get()); } return success; }
private boolean deleteArchivedInstants(List<HoodieInstant> archivedInstants) throws IOException { log.info("Deleting instants " + archivedInstants); boolean success = true; for (HoodieInstant archivedInstant : archivedInstants) { Path commitFile = new Path(metaClient.getMetaPath(), archivedInstant.getFileName()); try { if (metaClient.getFs().exists(commitFile)) { success &= metaClient.getFs().delete(commitFile, false); log.info("Archived and deleted instant file " + commitFile); } } catch (IOException e) { throw new HoodieIOException("Failed to delete archived instant " + archivedInstant, e); } } // Remove older meta-data from auxiliary path too Optional<HoodieInstant> latestCommitted = archivedInstants.stream() .filter(i -> { return i.isCompleted() && (i.getAction().equals(COMMIT_ACTION) || (i.getAction().equals(DELTA_COMMIT_ACTION))); }).max(Comparator.comparing(HoodieInstant::getTimestamp)); if (latestCommitted.isPresent()) { success &= deleteAllInstantsOlderorEqualsInAuxMetaFolder(latestCommitted.get()); } return success; }
.filterCompletedInstants(); Stream<HoodieInstant> instants = cleanAndRollbackTimeline.getInstants() .collect(Collectors.groupingBy(s -> s.getAction())).entrySet().stream().map(i -> { if (i.getValue().size() > maxCommitsToKeep) { return i.getValue().subList(0, i.getValue().size() - minCommitsToKeep);
.filterCompletedInstants(); Stream<HoodieInstant> instants = cleanAndRollbackTimeline.getInstants() .collect(Collectors.groupingBy(s -> s.getAction())).entrySet().stream().map(i -> { if (i.getValue().size() > maxCommitsToKeep) { return i.getValue().subList(0, i.getValue().size() - minCommitsToKeep);
}; switch (instant.getAction()) { case HoodieTimeline.COMMIT_ACTION: case HoodieTimeline.COMPACTION_ACTION: HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes( metaClient.getCommitTimeline().getInstantDetails( new HoodieInstant(true, instant.getAction(), instant.getTimestamp())) .get(), HoodieCommitMetadata.class); entry -> new HoodieInstant(true, entry.getValue().getAction(), entry.getValue().getTimestamp())).forEach(this.getActiveTimeline()::deleteInflight); logger
HoodieInstant instant = instants.get(i); HoodieCompactionPlan workload = null; if (!instant.getAction().equals(COMPACTION_ACTION)) { try {
HoodieArchivedMetaEntry archivedMetaWrapper = new HoodieArchivedMetaEntry(); archivedMetaWrapper.setCommitTime(hoodieInstant.getTimestamp()); switch (hoodieInstant.getAction()) { case HoodieTimeline.CLEAN_ACTION: { archivedMetaWrapper.setHoodieCleanMetadata(AvroUtils
HoodieArchivedMetaEntry archivedMetaWrapper = new HoodieArchivedMetaEntry(); archivedMetaWrapper.setCommitTime(hoodieInstant.getTimestamp()); switch (hoodieInstant.getAction()) { case HoodieTimeline.CLEAN_ACTION: { archivedMetaWrapper.setHoodieCleanMetadata(AvroUtils