/** * Gets the commit action type */ public String getCommitActionType() { switch (this.getTableType()) { case COPY_ON_WRITE: return HoodieActiveTimeline.COMMIT_ACTION; case MERGE_ON_READ: return HoodieActiveTimeline.DELTA_COMMIT_ACTION; default: throw new HoodieException( "Could not commit on unknown storage type " + this.getTableType()); } }
@CliAvailabilityIndicator({"compactions show all", "compaction show", "compaction run", "compaction schedule"}) public boolean isAvailable() { return (HoodieCLI.tableMetadata != null) && (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ); }
public static <T extends HoodieRecordPayload> HoodieTable<T> getHoodieTable( HoodieTableMetaClient metaClient, HoodieWriteConfig config, JavaSparkContext jsc) { switch (metaClient.getTableType()) { case COPY_ON_WRITE: return new HoodieCopyOnWriteTable<>(config, jsc); case MERGE_ON_READ: return new HoodieMergeOnReadTable<>(config, jsc); default: throw new HoodieException("Unsupported table type :" + metaClient.getTableType()); } }
public static <T extends HoodieRecordPayload> HoodieTable<T> getHoodieTable( HoodieTableMetaClient metaClient, HoodieWriteConfig config, JavaSparkContext jsc) { switch (metaClient.getTableType()) { case COPY_ON_WRITE: return new HoodieCopyOnWriteTable<>(config, jsc); case MERGE_ON_READ: return new HoodieMergeOnReadTable<>(config, jsc); default: throw new HoodieException("Unsupported table type :" + metaClient.getTableType()); } }
/** * Get the compacted commit timeline visible for this table */ public HoodieTimeline getCommitTimeline() { switch (this.getTableType()) { case COPY_ON_WRITE: case MERGE_ON_READ: // We need to include the parquet files written out in delta commits in tagging return getActiveTimeline().getCommitTimeline(); default: throw new HoodieException("Unsupported table type :" + this.getTableType()); } }
/** * Get the commit timeline visible for this table */ public HoodieTimeline getCommitsTimeline() { switch (this.getTableType()) { case COPY_ON_WRITE: return getActiveTimeline().getCommitTimeline(); case MERGE_ON_READ: // We need to include the parquet files written out in delta commits // Include commit action to be able to start doing a MOR over a COW dataset - no // migration required return getActiveTimeline().getCommitsTimeline(); default: throw new HoodieException("Unsupported table type :" + this.getTableType()); } }
/** * Get the commit + pending-compaction timeline visible for this table. * A RT filesystem view is constructed with this timeline so that file-slice after pending compaction-requested * instant-time is also considered valid. A RT file-system view for reading must then merge the file-slices before * and after pending compaction instant so that all delta-commits are read. */ public HoodieTimeline getCommitsAndCompactionTimeline() { switch (this.getTableType()) { case COPY_ON_WRITE: return getActiveTimeline().getCommitTimeline(); case MERGE_ON_READ: return getActiveTimeline().getCommitsAndCompactionTimeline(); default: throw new HoodieException("Unsupported table type :" + this.getTableType()); } }
deletePaths.add(dataFile.getFileStatus().getPath().toString()); if (hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ) {
/** * Obtain all the commits, compactions that have occurred on the timeline, whose instant times * could be fed into the datasource options. */ public static HoodieTimeline allCompletedCommitsCompactions(FileSystem fs, String basePath) { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(fs.getConf(), basePath, true); if (metaClient.getTableType().equals(HoodieTableType.MERGE_ON_READ)) { return metaClient.getActiveTimeline().getTimelineOfActions( Sets.newHashSet(HoodieActiveTimeline.COMMIT_ACTION, HoodieActiveTimeline.DELTA_COMMIT_ACTION)); } else { return metaClient.getCommitTimeline().filterCompletedInstants(); } } }
deletePaths.add(dataFile.getFileStatus().getPath().toString()); if (hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ) {
/** * Obtain all the commits, compactions that have occurred on the timeline, whose instant times * could be fed into the datasource options. */ public static HoodieTimeline allCompletedCommitsCompactions(FileSystem fs, String basePath) { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(fs.getConf(), basePath, true); if (metaClient.getTableType().equals(HoodieTableType.MERGE_ON_READ)) { return metaClient.getActiveTimeline().getTimelineOfActions( Sets.newHashSet(HoodieActiveTimeline.COMMIT_ACTION, HoodieActiveTimeline.DELTA_COMMIT_ACTION)); } else { return metaClient.getCommitTimeline().filterCompletedInstants(); } } }
HoodieHiveClient(HiveSyncConfig cfg, HiveConf configuration, FileSystem fs) { this.syncConfig = cfg; this.fs = fs; this.metaClient = new HoodieTableMetaClient(fs.getConf(), cfg.basePath, true); this.tableType = metaClient.getTableType(); LOG.info("Creating hive connection " + cfg.jdbcUrl); createHiveConnection(); try { this.client = new HiveMetaStoreClient(configuration); } catch (MetaException e) { throw new HoodieHiveSyncException("Failed to create HiveMetaStoreClient", e); } try { this.partitionValueExtractor = (PartitionValueExtractor) Class.forName( cfg.partitionValueExtractorClass).newInstance(); } catch (Exception e) { throw new HoodieHiveSyncException( "Failed to initialize PartitionValueExtractor class " + cfg.partitionValueExtractorClass, e); } activeTimeline = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(); }
if (hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ) {
.checkArgument(hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ, "HoodieRealtimeTableCompactor can only compact table of type " + HoodieTableType.MERGE_ON_READ + " and not " + hoodieTable.getMetaClient() .getTableType().name());
Path outputPath = new Path(outputPathStr); String output = ""; if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) { try { String sparkPropertiesPath = Utils.getDefaultPropertiesFile(
Path outputPath = new Path(outputPathStr); String output = ""; if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) { try { String sparkPropertiesPath = Utils.getDefaultPropertiesFile(
@CliCommand(value = "compaction schedule", help = "Schedule Compaction") public String scheduleCompact( @CliOption(key = "sparkMemory", unspecifiedDefaultValue = "1G", help = "Spark executor memory") final String sparkMemory) throws Exception { boolean initialized = HoodieCLI.initConf(); HoodieCLI.initFS(initialized); // First get a compaction instant time and pass it to spark launcher for scheduling compaction String compactionInstantTime = HoodieActiveTimeline.createNewCommitTime(); if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) { String sparkPropertiesPath = Utils.getDefaultPropertiesFile( scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties())); SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath); sparkLauncher.addAppArgs(SparkCommand.COMPACT_SCHEDULE.toString(), HoodieCLI.tableMetadata.getBasePath(), HoodieCLI.tableMetadata.getTableConfig().getTableName(), compactionInstantTime, sparkMemory); Process process = sparkLauncher.launch(); InputStreamConsumer.captureOutput(process); int exitCode = process.waitFor(); if (exitCode != 0) { return "Failed to run compaction for " + compactionInstantTime; } return "Compaction successfully completed for " + compactionInstantTime; } else { throw new Exception("Compactions can only be run for table type : MERGE_ON_READ"); } }
Path outputPath = new Path(outputPathStr); String output = null; if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) { try { String sparkPropertiesPath = Utils.getDefaultPropertiesFile(
Path outputPath = new Path(outputPathStr); String output = ""; if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) { try { String sparkPropertiesPath = Utils.getDefaultPropertiesFile(
HoodieCLI.initFS(initialized); if (HoodieCLI.tableMetadata.getTableType() == HoodieTableType.MERGE_ON_READ) { String sparkPropertiesPath = Utils.getDefaultPropertiesFile( scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));