congrats Icon
New! Tabnine Pro 14-day free trial
Start a free trial
Tabnine Logo
HoodieActiveTimeline.getInstantAuxiliaryDetails
Code IndexAdd Tabnine to your IDE (free)

How to use
getInstantAuxiliaryDetails
method
in
com.uber.hoodie.common.table.timeline.HoodieActiveTimeline

Best Java code snippets using com.uber.hoodie.common.table.timeline.HoodieActiveTimeline.getInstantAuxiliaryDetails (Showing top 9 results out of 315)

origin: uber/hudi

public static HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient,
  String compactionInstant) throws IOException {
 HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
   metaClient.getActiveTimeline().getInstantAuxiliaryDetails(
     HoodieTimeline.getCompactionRequestedInstant(compactionInstant)).get());
 return compactionPlan;
}
origin: uber/hudi

/**
 * Construction Compaction Plan from compaction instant
 */
private static HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient,
  String compactionInstant) throws IOException {
 HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
   metaClient.getActiveTimeline().getInstantAuxiliaryDetails(
     HoodieTimeline.getCompactionRequestedInstant(compactionInstant)).get());
 return compactionPlan;
}
origin: uber/hudi

/**
 * Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file.
 */
public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> writeStatuses,
  Optional<Map<String, String>> extraMetadata) throws IOException {
 HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(),
   config.getBasePath(), true);
 HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
 HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
 HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
   timeline.getInstantAuxiliaryDetails(HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get());
 // Merge extra meta-data passed by user with the one already in inflight compaction
 Optional<Map<String, String>> mergedMetaData = extraMetadata.map(m -> {
  Map<String, String> merged = new HashMap<>();
  Map<String, String> extraMetaDataFromInstantFile = compactionPlan.getExtraMetadata();
  if (extraMetaDataFromInstantFile != null) {
   merged.putAll(extraMetaDataFromInstantFile);
  }
  // Overwrite/Merge with the user-passed meta-data
  merged.putAll(m);
  return Optional.of(merged);
 }).orElseGet(() -> Optional.ofNullable(compactionPlan.getExtraMetadata()));
 commitCompaction(writeStatuses, table, compactionInstantTime, true, mergedMetaData);
}
origin: com.uber.hoodie/hoodie-client

/**
 * Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file.
 */
public void commitCompaction(String compactionInstantTime, JavaRDD<WriteStatus> writeStatuses,
  Optional<Map<String, String>> extraMetadata) throws IOException {
 HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(),
   config.getBasePath(), true);
 HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
 HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
 HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
   timeline.getInstantAuxiliaryDetails(HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get());
 // Merge extra meta-data passed by user with the one already in inflight compaction
 Optional<Map<String, String>> mergedMetaData = extraMetadata.map(m -> {
  Map<String, String> merged = new HashMap<>();
  Map<String, String> extraMetaDataFromInstantFile = compactionPlan.getExtraMetadata();
  if (extraMetaDataFromInstantFile != null) {
   merged.putAll(extraMetaDataFromInstantFile);
  }
  // Overwrite/Merge with the user-passed meta-data
  merged.putAll(m);
  return Optional.of(merged);
 }).orElseGet(() -> Optional.ofNullable(compactionPlan.getExtraMetadata()));
 commitCompaction(writeStatuses, table, compactionInstantTime, true, mergedMetaData);
}
origin: uber/hudi

   activeTimeline.getInstantAuxiliaryDetails(
     HoodieTimeline.getCompactionRequestedInstant(instant.getTimestamp())).get());
} catch (HoodieIOException ioe) {
workload = AvroUtils.deserializeCompactionPlan(activeTimeline.getInstantAuxiliaryDetails(
  HoodieTimeline.getCompactionRequestedInstant(instant.getTimestamp())).get());
origin: uber/hudi

/**
 * Perform compaction operations as specified in the compaction commit file
 *
 * @param compactionInstant  Compacton Instant time
 * @param activeTimeline Active Timeline
 * @param autoCommit         Commit after compaction
 * @return RDD of Write Status
 */
private JavaRDD<WriteStatus> runCompaction(
  HoodieInstant compactionInstant, HoodieActiveTimeline activeTimeline, boolean autoCommit) throws IOException {
 HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
   activeTimeline.getInstantAuxiliaryDetails(compactionInstant).get());
 // Mark instant as compaction inflight
 activeTimeline.transitionCompactionRequestedToInflight(compactionInstant);
 compactionTimer = metrics.getCompactionCtx();
 // Create a Hoodie table which encapsulated the commits and files visible
 HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(),
   config.getBasePath(), true);
 HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
 JavaRDD<WriteStatus> statuses = table.compact(jsc, compactionInstant.getTimestamp(), compactionPlan);
 // Force compaction action
 statuses.persist(config.getWriteStatusStorageLevel());
 // pass extra-metada so that it gets stored in commit file automatically
 commitCompaction(statuses, table, compactionInstant.getTimestamp(), autoCommit,
   Optional.ofNullable(compactionPlan.getExtraMetadata()));
 return statuses;
}
origin: uber/hudi

private void moveCompactionFromRequestedToInflight(String compactionInstantTime, HoodieWriteClient client,
  HoodieWriteConfig cfg) throws IOException {
 HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath());
 HoodieInstant compactionInstant = HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime);
 HoodieCompactionPlan workload = AvroUtils.deserializeCompactionPlan(
   metaClient.getActiveTimeline().getInstantAuxiliaryDetails(compactionInstant).get());
 metaClient.getActiveTimeline().transitionCompactionRequestedToInflight(compactionInstant);
 HoodieInstant instant = metaClient.getActiveTimeline().reload().filterPendingCompactionTimeline().getInstants()
   .filter(in -> in.getTimestamp().equals(compactionInstantTime)).findAny().get();
 assertTrue("Instant must be marked inflight", instant.isInflight());
}
origin: com.uber.hoodie/hoodie-client

/**
 * Perform compaction operations as specified in the compaction commit file
 *
 * @param compactionInstant  Compacton Instant time
 * @param activeTimeline Active Timeline
 * @param autoCommit         Commit after compaction
 * @return RDD of Write Status
 */
private JavaRDD<WriteStatus> runCompaction(
  HoodieInstant compactionInstant, HoodieActiveTimeline activeTimeline, boolean autoCommit) throws IOException {
 HoodieCompactionPlan compactionPlan = AvroUtils.deserializeCompactionPlan(
   activeTimeline.getInstantAuxiliaryDetails(compactionInstant).get());
 // Mark instant as compaction inflight
 activeTimeline.transitionCompactionRequestedToInflight(compactionInstant);
 compactionTimer = metrics.getCompactionCtx();
 // Create a Hoodie table which encapsulated the commits and files visible
 HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(),
   config.getBasePath(), true);
 HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
 JavaRDD<WriteStatus> statuses = table.compact(jsc, compactionInstant.getTimestamp(), compactionPlan);
 // Force compaction action
 statuses.persist(config.getWriteStatusStorageLevel());
 // pass extra-metada so that it gets stored in commit file automatically
 commitCompaction(statuses, table, compactionInstant.getTimestamp(), autoCommit,
   Optional.ofNullable(compactionPlan.getExtraMetadata()));
 return statuses;
}
origin: uber/hudi

HoodieActiveTimeline activeTimeline = HoodieCLI.tableMetadata.getActiveTimeline();
HoodieCompactionPlan workload = AvroUtils.deserializeCompactionPlan(
  activeTimeline.getInstantAuxiliaryDetails(
    HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime)).get());
com.uber.hoodie.common.table.timelineHoodieActiveTimelinegetInstantAuxiliaryDetails

Javadoc

BEGIN - COMPACTION RELATED META-DATA MANAGEMENT

Popular methods of HoodieActiveTimeline

  • getCommitTimeline
  • getCommitsTimeline
  • getCommitsAndCompactionTimeline
  • getInstantDetails
  • getTimelineOfActions
  • createNewCommitTime
  • reload
  • <init>
  • createInflight
  • filterPendingCompactionTimeline
  • getDeltaCommitTimeline
  • lastInstant
  • getDeltaCommitTimeline,
  • lastInstant,
  • saveAsComplete,
  • saveToCompactionRequested,
  • transitionCompactionRequestedToInflight,
  • deleteCompactionRequested,
  • deleteInflight,
  • filterInflightsExcludingCompaction,
  • getAllCommitsTimeline

Popular in Java

  • Making http post requests using okhttp
  • findViewById (Activity)
  • setRequestProperty (URLConnection)
  • getOriginalFilename (MultipartFile)
    Return the original filename in the client's filesystem.This may contain path information depending
  • Rectangle (java.awt)
    A Rectangle specifies an area in a coordinate space that is enclosed by the Rectangle object's top-
  • StringTokenizer (java.util)
    Breaks a string into tokens; new code should probably use String#split.> // Legacy code: StringTo
  • TimerTask (java.util)
    The TimerTask class represents a task to run at a specified time. The task may be run once or repeat
  • ConcurrentHashMap (java.util.concurrent)
    A plug-in replacement for JDK1.5 java.util.concurrent.ConcurrentHashMap. This version is based on or
  • Stream (java.util.stream)
    A sequence of elements supporting sequential and parallel aggregate operations. The following exampl
  • FileUtils (org.apache.commons.io)
    General file manipulation utilities. Facilities are provided in the following areas: * writing to a
  • Top 25 Plugins for Webstorm
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyStudentsTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now