private void setHiveQueueName() throws SQLException { this.conn.executeStatements("set " + MAPREDUCE_JOB_QUEUENAME + "=" + CompactionRunner.jobProperties.getProperty(HIVE_QUEUE_NAME, "default")); }
/** * Create a new {@link HiveJdbcConnector} using the specified Hive server version. * @param hiveServerVersion is the Hive server version to use * @return a HiveJdbcConnector with the specified hiveServerVersion * @throws SQLException */ @SuppressWarnings("resource") public static HiveJdbcConnector newEmbeddedConnector(int hiveServerVersion) throws SQLException { return new HiveJdbcConnector().withHiveServerVersion(hiveServerVersion).withHiveEmbeddedConnection(); }
/*** * Executes the given SQL statements. * * @param statements SQL statements to be executed. * @throws SQLException if any issue in executing any statement. */ public void executeStatements(String... statements) throws SQLException { Preconditions.checkNotNull(this.conn, "The Hive connection must be set before any queries can be run"); for (String statement : statements) { if (isSimulate) { LOG.info("[SIMULATE MODE] STATEMENT NOT RUN: " + choppedStatement(statement)); } else { LOG.info("RUNNING STATEMENT: " + choppedStatement(statement)); this.stmt.execute(statement); } } }
/** * Create a new {@link HiveJdbcConnector} based on the configs in a {@link Properties} object * @param compactRunProps contains the configuration keys to construct the {@link HiveJdbcConnector} * @throws SQLException if there is a problem setting up the JDBC connection * @return */ public static HiveJdbcConnector newConnectorWithProps(Properties compactRunProps) throws SQLException { boolean isSimulate = Boolean.valueOf(compactRunProps.getProperty(HIVE_EXECUTION_SIMULATE)); HiveJdbcConnector hiveJdbcConnector = new HiveJdbcConnector(isSimulate); // Set the Hive Server type hiveJdbcConnector.withHiveServerVersion( Integer.parseInt(compactRunProps.getProperty(HIVESERVER_VERSION, DEFAULT_HIVESERVER_VERSION))); // Add the Hive Site Dir to the classpath if (compactRunProps.containsKey(HIVESITE_DIR)) { HiveJdbcConnector.addHiveSiteDirToClasspath(compactRunProps.getProperty(HIVESITE_DIR)); } // Set and create the Hive JDBC connection if (compactRunProps.containsKey(HIVESERVER_CONNECTION_STRING)) { hiveJdbcConnector.withHiveConnectionFromUrl(compactRunProps.getProperty(HIVESERVER_CONNECTION_STRING)); } else if (compactRunProps.containsKey(HIVESERVER_URL)) { hiveJdbcConnector.withHiveConnectionFromUrlUserPassword(compactRunProps.getProperty(HIVESERVER_URL), compactRunProps.getProperty(HIVESERVER_USER), PasswordManager.getInstance(compactRunProps).readPassword(compactRunProps.getProperty(HIVESERVER_PASSWORD))); } else { hiveJdbcConnector.withHiveEmbeddedConnection(); } // Set Hive properties hiveJdbcConnector.setHiveProperties(compactRunProps); return hiveJdbcConnector; }
HiveJdbcConnector hiveJdbcConnector = closer.register(HiveJdbcConnector.newConnectorWithProps(props)); for (String query : queries) { String hiveOutput = "hiveConversionValidationOutput_" + UUID.randomUUID().toString(); try { if (this.hiveSettings.size() > 0) { hiveJdbcConnector.executeStatements(this.hiveSettings.toArray(new String[this.hiveSettings.size()])); hiveJdbcConnector.executeStatements("SET hive.exec.compress.output=false","SET hive.auto.convert.join=false", query); FileStatus[] fileStatusList = this.fs.listStatus(hiveTempDir); List<FileStatus> files = new ArrayList<>();
@Override public DataWriter<QueryBasedHiveConversionEntity> build() throws IOException { try { return new HiveQueryExecutionWriter(HiveJdbcConnector.newConnectorWithProps(this.destination.getProperties().getProperties()), this.destination.getProperties()); } catch (SQLException e) { throw new RuntimeException(e); } } }
HiveJdbcConnector hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(props); statement = hiveJdbcConnector.getConnection().createStatement();
@Override public void close() throws IOException { this.hiveJdbcConnector.close(); }
@Override public void compact() throws IOException { checkSchemaCompatibility(); Closer closer = Closer.create(); try { this.conn = closer.register(HiveJdbcConnector.newConnectorWithProps(CompactionRunner.properties)); setHiveParameters(); createTables(); HiveTable mergedDelta = mergeDeltas(); HiveManagedTable notUpdated = getNotUpdatedRecords(this.snapshot, mergedDelta); unionNotUpdatedRecordsAndDeltas(notUpdated, mergedDelta); } catch (SQLException e) { LOG.error("SQLException during compaction: " + e.getMessage()); throw new RuntimeException(e); } catch (IOException e) { LOG.error("IOException during compaction: " + e.getMessage()); throw new RuntimeException(e); } catch (RuntimeException e) { LOG.error("Runtime Exception during compaction: " + e.getMessage()); throw e; } finally { try { deleteTmpFiles(); } finally { closer.close(); } } }
@Override public void close() throws IOException { this.avroSchemaManager.cleanupTempSchemas(); this.hiveJdbcConnector.close(); }
private void setHiveDbName() throws SQLException { this.conn.executeStatements("use " + CompactionRunner.jobProperties.getProperty(HIVE_DB_NAME, "default")); }
public HiveConvertPublisher(State state) throws IOException { super(state); this.avroSchemaManager = new AvroSchemaManager(FileSystem.get(HadoopUtils.newConfiguration()), state); this.metricContext = Instrumented.getMetricContext(state, HiveConvertPublisher.class); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, EventConstants.CONVERSION_NAMESPACE).build(); Configuration conf = new Configuration(); Optional<String> uri = Optional.fromNullable(this.state.getProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI)); if (uri.isPresent()) { this.fs = FileSystem.get(URI.create(uri.get()), conf); } else { this.fs = FileSystem.get(conf); } try { this.hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(state.getProperties()); } catch (SQLException e) { throw new RuntimeException(e); } this.watermarker = GobblinConstructorUtils.invokeConstructor( HiveSourceWatermarkerFactory.class, state.getProp(HiveSource.HIVE_SOURCE_WATERMARKER_FACTORY_CLASS_KEY, HiveSource.DEFAULT_HIVE_SOURCE_WATERMARKER_FACTORY_CLASS)).createFromState(state); this.pool = HiveMetastoreClientPool.get(state.getProperties(), Optional.fromNullable(state.getProperties().getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); }
private void setHiveInputSplitSize() throws SQLException { boolean splitSizeSpecified = CompactionRunner.jobProperties.containsKey(HIVE_INPUT_SPLIT_SIZE); if (splitSizeSpecified) { this.conn.executeStatements( "set " + MAPRED_MIN_SPLIT_SIZE + "=" + CompactionRunner.jobProperties.getProperty(HIVE_INPUT_SPLIT_SIZE)); } }
private void setNumberOfReducers() throws SQLException { boolean numOfReducersSpecified = CompactionRunner.jobProperties.containsKey(MAPREDUCE_JOB_NUM_REDUCERS); if (numOfReducersSpecified) { this.conn.executeStatements("set " + MAPREDUCE_JOB_REDUCES + "=" + CompactionRunner.jobProperties.getProperty(MAPREDUCE_JOB_NUM_REDUCERS)); } }
private void setHiveMapjoin() throws SQLException { boolean useMapjoin = Boolean.parseBoolean(CompactionRunner.jobProperties.getProperty(HIVE_USE_MAPJOIN, "false")); boolean smallTableSizeSpecified = CompactionRunner.jobProperties.containsKey(HIVE_MAPJOIN_SMALLTABLE_FILESIZE); if (useMapjoin && smallTableSizeSpecified) { this.conn.executeStatements("set " + HIVE_AUTO_CONVERT_JOIN + "=true"); this.conn.executeStatements("set " + HIVE_MAPJOIN_SMALLTABLE_FILESIZE + "=" + CompactionRunner.jobProperties.getProperty(HIVE_MAPJOIN_SMALLTABLE_FILESIZE)); } }
private void executeQueries(List<String> queries) { if (null == queries || queries.size() == 0) { return; } try { this.hiveJdbcConnector.executeStatements(queries.toArray(new String[queries.size()])); } catch (SQLException e) { throw new RuntimeException(e); } }
public void dropTable(HiveJdbcConnector conn, String jobId) throws SQLException { String dropTableStmt = String.format(DROP_TABLE_STMT, getNameWithJobId(jobId)); conn.executeStatements(dropTableStmt); }
@Override public void createTable(HiveJdbcConnector conn, String jobID) throws SQLException { String tableName = getNameWithJobId(jobID); String dropTableStmt = String.format(DROP_TABLE_STMT, tableName); String hdfsUri = HdfsIO.getHdfsUri(); String createTableStmt = String.format(CREATE_TABLE_STMT, tableName, hdfsUri + this.dataLocationInHdfs, hdfsUri + this.schemaLocationInHdfs); conn.executeStatements(dropTableStmt, createTableStmt); }
public void createTable(HiveJdbcConnector conn, String jobId, String tableType) throws SQLException { String tableName = getNameWithJobId(jobId); String dropTableStmt = String.format(DROP_TABLE_STMT, tableName); StringBuilder sb = new StringBuilder().append("CREATE "); sb.append(tableType + " "); sb.append(tableName); sb.append('('); for (int i = 0; i < this.attributes.size(); i++) { sb.append(this.attributes.get(i).name() + " " + this.attributes.get(i).type()); if (i != this.attributes.size() - 1) { sb.append(", "); } } sb.append(")"); String createTableStmt = sb.toString(); conn.executeStatements(dropTableStmt, createTableStmt); }
private void insertFirstDeltaIntoMergedDelta(HiveManagedTable mergedDelta) throws SQLException { String insertStmt = "INSERT OVERWRITE TABLE " + mergedDelta.getNameWithJobId(this.jobId) + " SELECT * FROM " + this.deltas.get(0).getNameWithJobId(this.jobId); this.conn.executeStatements(insertStmt); }