protected static HiveMetastoreClientPool createClientPool(Properties properties) throws IOException { return HiveMetastoreClientPool.get(properties, Optional.fromNullable(properties.getProperty(HIVE_METASTORE_URI_KEY))); }
public static HivePartitionDataset findDataset(String completePartitionName, State prop) throws IOException { synchronized (lock) { List<String> partitionList = AT_SPLITTER.splitToList(completePartitionName); Preconditions.checkArgument(partitionList.size() == 3, "Invalid partition name"); if (!pool.isPresent()) { pool = Optional.of(HiveMetastoreClientPool.get(new Properties(), Optional.fromNullable(new Properties().getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY)))); } try (AutoReturnableObject<IMetaStoreClient> client = pool.get().getClient()) { Table table = new Table(client.get().getTable(partitionList.get(0), partitionList.get(1))); Partition partition = new Partition(table, client.get().getPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2))); return new HivePartitionDataset(partition); } catch (TException | HiveException e) { throw new IOException(e); } } }
private LocalHiveMetastoreTestUtils() throws IOException { try { FileUtils.deleteDirectory(new File("metastore_db")); } catch (IOException e) { e.printStackTrace(); } this.localMetastoreClient = HiveMetastoreClientPool.get(new Properties(), Optional.<String>absent()).getClient().get(); }
public ComplianceRetentionJob(Properties properties) { super(properties); try { this.pool = HiveMetastoreClientPool .get(properties, Optional.fromNullable(properties.getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); initDatasetFinder(properties); ProxyUtils.cancelTokens(new State(properties)); } catch (InterruptedException | TException | IOException e) { Throwables.propagate(e); } }
HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(props, Optional.fromNullable(props.getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) {
EventSubmitter.submit(Optional.of(this.eventSubmitter), EventConstants.VALIDATION_SETUP_EVENT); this.pool = HiveMetastoreClientPool.get(props, Optional.fromNullable(props.getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); Preconditions.checkArgument(props.containsKey(VALIDATION_TYPE_KEY), "Missing property " + VALIDATION_TYPE_KEY); this.validationType = ValidationType.valueOf(props.getProperty(VALIDATION_TYPE_KEY));
HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(), Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) {
try { this.pool = HiveMetastoreClientPool.get(state.getProperties(), Optional.fromNullable(state.getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); } catch (IOException e) {
HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(), Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) {
HiveSourceWatermarkerFactory.class, state.getProp(HiveSource.HIVE_SOURCE_WATERMARKER_FACTORY_CLASS_KEY, HiveSource.DEFAULT_HIVE_SOURCE_WATERMARKER_FACTORY_CLASS)).createFromState(state); this.pool = HiveMetastoreClientPool.get(state.getProperties(), Optional.fromNullable(state.getProperties().getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY)));
private HiveDataset getHiveDataset(String tableString, FileSystem fs, State state) throws IOException { try { HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(state.getProperties(), Optional.fromNullable(state.getProp(HIVE_METASTORE_URI_KEY))); List<String> tokens = Splitter.on(".").splitToList(tableString); DbAndTable sourceDbAndTable = new DbAndTable(tokens.get(0), tokens.get(1)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Table sourceTable = new Table(client.get().getTable(sourceDbAndTable.getDb(), sourceDbAndTable.getTable())); return new HiveDataset(fs, pool, sourceTable, ConfigUtils.propertiesToConfig(state.getProperties())); } } catch (TException exc) { throw new RuntimeException(exc); } }
public HiveBaseExtractor(WorkUnitState state) throws IOException { if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) { return; } this.hiveWorkUnit = new HiveWorkUnit(state.getWorkunit()); this.hiveDataset = hiveWorkUnit.getHiveDataset(); this.dbName = hiveDataset.getDbAndTable().getDb(); this.tableName = hiveDataset.getDbAndTable().getTable(); this.pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(), Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); }
@BeforeClass public void setup() throws Exception { this.jdbcConnector = HiveJdbcConnector.newEmbeddedConnector(2); this.dataFile = new File(getClass().getClassLoader().getResource("hiveMaterializerTest/source/").toURI()); this.localHiveMetastore.dropDatabaseIfExists(this.dbName); this.localHiveMetastore.createTestDb(this.dbName); this.jdbcConnector.executeStatements( String.format("CREATE EXTERNAL TABLE %s.%s (id STRING, name String) PARTITIONED BY (%s String) " + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE", this.dbName, this.sourceTableName, this.partitionColumn), String.format("ALTER TABLE %s.%s ADD PARTITION (part = 'part1') LOCATION '%s'", this.dbName, this.sourceTableName, this.dataFile.getAbsolutePath() + "/part1"), String.format("ALTER TABLE %s.%s ADD PARTITION (part = 'part2') LOCATION '%s'", this.dbName, this.sourceTableName, this.dataFile.getAbsolutePath() + "/part2")); List<List<String>> allTable = executeStatementAndGetResults(this.jdbcConnector, String.format("SELECT * FROM %s.%s", this.dbName, this.sourceTableName), 3); Assert.assertEquals(allTable.size(), 8); List<List<String>> part1 = executeStatementAndGetResults(this.jdbcConnector, String.format("SELECT * FROM %s.%s WHERE %s='part1'", this.dbName, this.sourceTableName, this.partitionColumn), 3); Assert.assertEquals(part1.size(), 4); this.pool = HiveMetastoreClientPool.get(new Properties(), Optional.absent()); Table table; try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { table = new Table(client.get().getTable(this.dbName, this.sourceTableName)); } this.dataset = new HiveDataset(FileSystem.getLocal(new Configuration()), pool, table, new Properties()); }
this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY)); this.targetClientPool = HiveMetastoreClientPool.get(this.dataset.getProperties(), this.targetURI); this.targetDatabase = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_DATABASE_KEY)) .or(this.dataset.table.getDbName());
private HiveDataset getHiveDataset(String tableString, FileSystem fs, State state) throws IOException { try { HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(state.getProperties(), Optional.fromNullable(state.getProp(HIVE_METASTORE_URI_KEY))); List<String> tokens = Splitter.on(".").splitToList(tableString); DbAndTable sourceDbAndTable = new DbAndTable(tokens.get(0), tokens.get(1)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Table sourceTable = new Table(client.get().getTable(sourceDbAndTable.getDb(), sourceDbAndTable.getTable())); return new HiveDataset(fs, pool, sourceTable, ConfigUtils.propertiesToConfig(state.getProperties())); } } catch (TException exc) { throw new RuntimeException(exc); } }
public HiveMaterializerQueryGenerator(WorkUnitState workUnitState) throws IOException { this.fs = HiveSource.getSourceFs(workUnitState); this.pool = HiveMetastoreClientPool.get(workUnitState.getJobState().getProperties(), Optional.fromNullable(workUnitState.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); this.workUnitState = workUnitState; this.workUnit = new HiveWorkUnit(workUnitState.getWorkunit()); this.outputTableMetadata = HiveMaterializer.parseStageableTableMetadata(this.workUnit); this.outputDatabaseName = outputTableMetadata.getDestinationDbName(); this.outputTableName = outputTableMetadata.getDestinationTableName(); this.outputDataLocation = HiveConverterUtils.getOutputDataLocation(outputTableMetadata.getDestinationDataPath()); this.destinationTableMeta = HiveConverterUtils.getDestinationTableMeta(this.outputTableMetadata.getDestinationDbName(), this.outputTableMetadata.getDestinationTableName(), workUnitState.getProperties()).getLeft(); this.stagingTableName = HiveConverterUtils.getStagingTableName(this.outputTableMetadata.getDestinationStagingTableName()); this.stagingDataLocation = HiveConverterUtils.getStagingDataLocation(this.outputTableMetadata.getDestinationDataPath(), this.stagingTableName); }
public HiveMetaStoreBasedRegister(State state, Optional<String> metastoreURI) throws IOException { super(state); this.optimizedChecks = state.getPropAsBoolean(this.OPTIMIZED_CHECK_ENABLED, true); GenericObjectPoolConfig config = new GenericObjectPoolConfig(); config.setMaxTotal(this.props.getNumThreads()); config.setMaxIdle(this.props.getNumThreads()); this.clientPool = HiveMetastoreClientPool.get(this.props.getProperties(), metastoreURI); this.metricContext = GobblinMetricsRegistry.getInstance().getMetricContext(state, HiveMetaStoreBasedRegister.class, GobblinMetrics.getCustomTagsFromState(state)); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, "org.apache.gobblin.hive.HiveMetaStoreBasedRegister").build(); }
protected static HiveMetastoreClientPool createClientPool(Properties properties) throws IOException { return HiveMetastoreClientPool.get(properties, Optional.fromNullable(properties.getProperty(HIVE_METASTORE_URI_KEY))); }
public HiveBaseExtractor(WorkUnitState state) throws IOException { if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) { return; } this.hiveWorkUnit = new HiveWorkUnit(state.getWorkunit()); this.hiveDataset = hiveWorkUnit.getHiveDataset(); this.dbName = hiveDataset.getDbAndTable().getDb(); this.tableName = hiveDataset.getDbAndTable().getTable(); this.pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(), Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); }
public HiveMetaStoreBasedRegister(State state, Optional<String> metastoreURI) throws IOException { super(state); this.optimizedChecks = state.getPropAsBoolean(this.OPTIMIZED_CHECK_ENABLED, true); GenericObjectPoolConfig config = new GenericObjectPoolConfig(); config.setMaxTotal(this.props.getNumThreads()); config.setMaxIdle(this.props.getNumThreads()); this.clientPool = HiveMetastoreClientPool.get(this.props.getProperties(), metastoreURI); this.metricContext = GobblinMetricsRegistry.getInstance().getMetricContext(state, HiveMetaStoreBasedRegister.class, GobblinMetrics.getCustomTagsFromState(state)); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, "org.apache.gobblin.hive.HiveMetaStoreBasedRegister").build(); }