/** * Helper method to initialize a given path, as a given storage type and table name */ public static HoodieTableMetaClient initTableType(Configuration hadoopConf, String basePath, HoodieTableType tableType, String tableName, String payloadClassName) throws IOException { Properties properties = new Properties(); properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tableName); properties.setProperty(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, tableType.name()); if (tableType == MERGE_ON_READ) { properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, payloadClassName); } return HoodieTableMetaClient.initializePathAsHoodieDataset(hadoopConf, basePath, properties); }
/** * Helper method to initialize a dataset, with given basePath, tableType, name, archiveFolder */ public static HoodieTableMetaClient initTableType(Configuration hadoopConf, String basePath, String tableType, String tableName, String archiveLogFolder) throws IOException { HoodieTableType type = HoodieTableType.valueOf(tableType); Properties properties = new Properties(); properties.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tableName); properties.put(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, type.name()); properties.put(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, archiveLogFolder); return HoodieTableMetaClient.initializePathAsHoodieDataset(hadoopConf, basePath, properties); }
/** * It initializes hoodie dataset * @param fs {@link FileSystem} * @param hoodieConf {@link HoodieConfiguration} * @throws IOException */ public static void initHoodieDataset(@NonNull final FileSystem fs, @NonNull final HoodieConfiguration hoodieConf) throws IOException { final Path hoodieMetaFolder = new Path(hoodieConf.getBasePath(), HoodieTableMetaClient.METAFOLDER_NAME); final Path hoodiePropertiesFile = new Path(hoodieMetaFolder.toString(), HoodieTableConfig.HOODIE_PROPERTIES_FILE); if (!fs.exists(hoodiePropertiesFile)) { HoodieTableMetaClient .initializePathAsHoodieDataset(FSUtils.getFs(hoodieConf.getConf()), hoodieConf.getBasePath(), hoodieConf.getHoodieInitProperties()); } }
public static HoodieTableMetaClient initTableType(Configuration hadoopConf, String basePath, HoodieTableType tableType) throws IOException { Properties properties = new Properties(); properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, RAW_TRIPS_TEST_NAME); properties.setProperty(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, tableType.name()); properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, HoodieAvroPayload.class.getName()); return HoodieTableMetaClient.initializePathAsHoodieDataset(hadoopConf, basePath, properties); }
@VisibleForTesting protected int dataImport(JavaSparkContext jsc) throws IOException { try { if (fs.exists(new Path(cfg.targetPath))) { // cleanup target directory. fs.delete(new Path(cfg.targetPath), true); } //Get schema. String schemaStr = UtilHelpers.parseSchema(fs, cfg.schemaFile); // Initialize target hoodie table. Properties properties = new Properties(); properties.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, cfg.tableName); properties.put(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, cfg.tableType); HoodieTableMetaClient .initializePathAsHoodieDataset(jsc.hadoopConfiguration(), cfg.targetPath, properties); HoodieWriteClient client = UtilHelpers.createHoodieClient(jsc, cfg.targetPath, schemaStr, cfg.parallelism, Optional.empty()); JavaRDD<HoodieRecord<HoodieRecordPayload>> hoodieRecords = buildHoodieRecordsForImport(jsc, schemaStr); // Get instant time. String instantTime = client.startCommit(); JavaRDD<WriteStatus> writeResponse = load(client, instantTime, hoodieRecords); return UtilHelpers.handleErrors(jsc, instantTime, writeResponse); } catch (Throwable t) { logger.error("Error occurred.", t); } return -1; }
@VisibleForTesting protected int dataImport(JavaSparkContext jsc) throws IOException { try { if (fs.exists(new Path(cfg.targetPath))) { // cleanup target directory. fs.delete(new Path(cfg.targetPath), true); } //Get schema. String schemaStr = UtilHelpers.parseSchema(fs, cfg.schemaFile); // Initialize target hoodie table. Properties properties = new Properties(); properties.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, cfg.tableName); properties.put(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, cfg.tableType); HoodieTableMetaClient .initializePathAsHoodieDataset(jsc.hadoopConfiguration(), cfg.targetPath, properties); HoodieWriteClient client = UtilHelpers.createHoodieClient(jsc, cfg.targetPath, schemaStr, cfg.parallelism, Optional.empty()); JavaRDD<HoodieRecord<HoodieRecordPayload>> hoodieRecords = buildHoodieRecordsForImport(jsc, schemaStr); // Get instant time. String instantTime = client.startCommit(); JavaRDD<WriteStatus> writeResponse = load(client, instantTime, hoodieRecords); return UtilHelpers.handleErrors(jsc, instantTime, writeResponse); } catch (Throwable t) { logger.error("Error occurred.", t); } return -1; }