private boolean checkConcurrency() { boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); if (!supportConcurrency) { LOG.info("Concurrency mode is disabled, not creating a lock manager"); return false; } return true; }
private static boolean isAllowed(Configuration conf, ConfVars setting) { String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, (String)null); return (mode != null) ? !"strict".equals(mode) : !HiveConf.getBoolVar(conf, setting); } }
public static boolean isAuthorizationEnabled(Configuration conf) { if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) { return false; } // If the V2 api of authorizer in use, the session state getAuthorizer return null. // Here we disable authorization if we use V2 api or the DefaultHiveAuthorizationProvider // The additional authorization checks happening in hcatalog are designed to // work with storage based authorization (on client side). It should not try doing // additional checks if a V2 authorizer or DefaultHiveAuthorizationProvider is in use. // The recommended configuration is to use storage based authorization in metastore server. // However, if user define a custom V1 authorization, it will be honored. if (SessionState.get().getAuthorizer() == null || DefaultHiveAuthorizationProvider.class.getName().equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER))) { LOG.info("Metastore authorizer is skipped for V2 authorizer or" + " DefaultHiveAuthorizationProvider"); return false; } return true; } }
private void setVertexParallelismAndRootInputSpec(Map<String, InputSpecUpdate> rootInputSpecUpdate) throws IOException { if (numInputsAffectingRootInputSpecUpdate != numInputsSeenSoFar) { return; } LOG.info("Setting vertex parallelism since we have seen all inputs."); boolean generateConsistentSplits = HiveConf.getBoolVar( conf, HiveConf.ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS); LOG.info("GenerateConsistenSplitsInHive=" + generateConsistentSplits); context.setVertexParallelism(taskCount, VertexLocationHint.create(grouper .createTaskLocationHints(finalSplits.toArray(new InputSplit[finalSplits.size()]), generateConsistentSplits)), emMap, rootInputSpecUpdate); finalSplits.clear(); }
private List<QueryLifeTimeHook> loadQueryHooks() throws Exception { List<QueryLifeTimeHook> hooks = new ArrayList<>(); if (conf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) { hooks.add(new MetricsQueryLifeTimeHook()); } List<QueryLifeTimeHook> propertyDefinedHoooks = getHooks(ConfVars.HIVE_QUERY_LIFETIME_HOOKS, QueryLifeTimeHook.class); if (propertyDefinedHoooks != null) { Iterables.addAll(hooks, propertyDefinedHoooks); } return hooks; }
private void analyzeAlterTableArchive(String[] qualified, CommonTree ast, boolean isUnArchive) throws SemanticException { if (!conf.getBoolVar(HiveConf.ConfVars.HIVEARCHIVEENABLED)) { throw new SemanticException(ErrorMsg.ARCHIVE_METHODS_DISABLED.getMsg()); inputs.add(new ReadEntity(tab)); if (partSpecs.size() > 1) { throw new SemanticException(isUnArchive ? ErrorMsg.UNARCHIVE_ON_MULI_PARTS.getMsg() : ErrorMsg.ARCHIVE_ON_MULI_PARTS.getMsg()); if (partSpecs.size() == 0) { throw new SemanticException(ErrorMsg.ARCHIVE_ON_TABLE.getMsg()); getDotName(qualified), partSpec, (isUnArchive ? AlterTableTypes.UNARCHIVE : AlterTableTypes.ARCHIVE)); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), archiveDesc)));
@Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} COMMIT_TXN message : {}", fromEventId(), eventMessageAsJSON); String payload = eventMessageAsJSON; if (!withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) { List<WriteEventInfo> writeEventInfoList = HiveMetaStore.HMSHandler.getMSForConf(withinContext.hiveConf). getAllWriteEventInfo(eventMessage.getTxnId(), contextDbName, contextTableName); int numEntry = (writeEventInfoList != null ? writeEventInfoList.size() : 0); if (numEntry != 0) { eventMessage.addWriteEventInfo(writeEventInfoList); qlPtns.add(new org.apache.hadoop.hive.ql.metadata.Partition(qlMdTable, eventMessage.getPartitionObj(idx))); filesTobeAdded.add(Lists.newArrayList( ReplChangeManager.getListFromSeparatedString(eventMessage.getFiles(idx))));
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("getSplits started"); } Configuration conf = job; if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED)) { // Create HiveConf once, since this is expensive. conf = new HiveConf(conf, OrcInputFormat.class); } List<OrcSplit> result = generateSplitsInfo(conf, new Context(conf, numSplits, createExternalCaches())); if (LOG.isDebugEnabled()) { LOG.debug("getSplits finished"); } return result.toArray(new InputSplit[result.size()]); }
ColStatistics cs = statistics.getColumnStatisticsFromColName(col); if (cs != null) { distinctVals.add(cs.getCountDistint()); distinctVal = 2L; } else { if (joinKeys.size() > 1 && conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_CORRELATED_MULTI_KEY_JOINS)) { distinctVal = Collections.max(distinctVals); } else {
@SuppressWarnings("unchecked") private void analyzeDropMacro(ASTNode ast) throws SemanticException { String functionName = ast.getChild(0).getText(); boolean ifExists = (ast.getFirstChildWithType(TOK_IFEXISTS) != null); // we want to signal an error if the function doesn't exist and we're // configured not to ignore this boolean throwException = !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT); // Temp macros are not allowed to have qualified names. if (FunctionUtils.isQualifiedFunctionName(functionName)) { throw new SemanticException("Temporary macro name cannot be a qualified name."); } if (throwException && FunctionRegistry.getFunctionInfo(functionName) == null) { throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(functionName)); } DropMacroDesc desc = new DropMacroDesc(functionName); rootTasks.add(TaskFactory.get(new FunctionWork(desc))); addEntities(); }
public void initialize() { if (initialized) { return; } initialized = true; queryHooks.addAll(loadHooksFromConf(HiveConf.ConfVars.HIVE_QUERY_LIFETIME_HOOKS, QueryLifeTimeHook.class)); saHooks.addAll(loadHooksFromConf(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, HiveSemanticAnalyzerHook.class)); driverRunHooks.addAll(loadHooksFromConf(HiveConf.ConfVars.HIVE_DRIVER_RUN_HOOKS, HiveDriverRunHook.class)); preExecHooks.addAll(loadHooksFromConf(HiveConf.ConfVars.PREEXECHOOKS, ExecuteWithHookContext.class)); postExecHooks.addAll(loadHooksFromConf(HiveConf.ConfVars.POSTEXECHOOKS, ExecuteWithHookContext.class)); onFailureHooks.addAll(loadHooksFromConf(HiveConf.ConfVars.ONFAILUREHOOKS, ExecuteWithHookContext.class)); if (conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED)) { queryHooks.add(new MetricsQueryLifeTimeHook()); } }
if (HiveConf.getBoolVar(pCtx.getConf(),HiveConf.ConfVars.HIVECONVERTJOIN)) { findPossibleAutoConvertedJoinOperators(); LOG.info("Abort. Reasons are ..."); for (String reason : corrCtx.getAbortReasons()) { LOG.info("-- " + reason); LOG.info("Begain query plan transformation based on intra-query correlations. " + corrCtx.getCorrelations().size() + " correlation(s) to be applied"); for (IntraQueryCorrelation correlation : corrCtx.getCorrelations()) { QueryPlanTreeTransformation.applyCorrelation(pCtx, corrCtx, correlation);
if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN)) { resolvers.add(new CommonJoinResolver()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_TOMAPJOIN)) { resolvers.add(new SortMergeJoinResolver()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVESKEWJOIN)) { resolvers.add(new SkewJoinResolver()); resolvers.add(new MapJoinResolver()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) { resolvers.add(new MetadataOnlyOptimizer()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVENULLSCANOPTIMIZE)) { resolvers.add(new NullScanOptimizer()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVESAMPLINGFORORDERBY)) { resolvers.add(new SamplingOptimizer()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_INFER_BUCKET_SORT)) { resolvers.add(new BucketingSortingInferenceOptimizer()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_CHECK_CROSS_PRODUCT)) { resolvers.add(new CrossProductHandler()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) || HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE).equalsIgnoreCase( "enable")) {
@Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ALTER_PARTITION message : {}", fromEventId(), eventMessageAsJSON); // We do not dump partitions during metadata only bootstrap dump (See TableExport // .getPartitions(), for bootstrap dump we pass tableSpec with TABLE_ONLY set.). So don't // dump partition related events for metadata-only dump. if (withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) { return; } Table qlMdTable = new Table(tableObject); if (!Utils.shouldReplicate(withinContext.replicationSpec, qlMdTable, withinContext.hiveConf)) { return; } if (Scenario.ALTER == scenario) { withinContext.replicationSpec.setIsMetadataOnly(true); List<Partition> partitions = new ArrayList<>(); partitions.add(new Partition(qlMdTable, after)); Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME); EximUtil.createExportDump( metaDataPath.getFileSystem(withinContext.hiveConf), metaDataPath, qlMdTable, partitions, withinContext.replicationSpec, withinContext.hiveConf); } DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(eventMessageAsJSON); dmd.write(); }
String onClauseAsString, Table targetTable, boolean onlyHaveWhenNotMatchedClause) throws SemanticException { if (!conf.getBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK)) { LOG.info("Merge statement cardinality violation check is disabled: " + HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK.varname); return false; table.setSerializationLib(format.getSerde()); List<FieldSchema> fields = new ArrayList<FieldSchema>(); fields.add(new FieldSchema("val", "int", null)); table.setFields(fields); table.setDataLocation(Warehouse.getDnsPath(new Path(SessionState.get().getTempTableSpace(), throw new SemanticException(e.getMessage(), e);
String validationQuery, boolean vectorize, String... records) throws Exception { AcidUtils.Directory dir = AcidUtils.getAcidState(partitionPath, conf, getTransactionContext(conf)); Assert.assertEquals(0, dir.getObsolete().size()); Assert.assertEquals(0, dir.getOriginalFiles().size()); List<AcidUtils.ParsedDelta> current = dir.getCurrentDirectories(); System.out.println("Files found: "); System.out.println(pd.getPath().toString()); Assert.assertEquals(numExpectedFiles, current.size()); boolean isVectorizationEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED); if(vectorize) { conf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); String currStrategy = conf.getVar(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY); for(String strategy : ((Validator.StringSet)HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.getValidator()).getExpected()) {
boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE) || conf.getBoolVar(HiveConf.ConfVars.HIVEEXIMTESTMODE); URI uri = new Path(dcPath).toUri(); FileSystem fs = FileSystem.get(uri, conf); String path = uri.getPath(); LOG.info("Path before norm :" + path); LOG.info("Scheme:" + scheme + ", authority:" + authority + ", path:" + path); Collection<String> eximSchemes = conf.getStringCollection(HiveConf.ConfVars.HIVE_EXIM_URI_SCHEME_WL.varname); if (!eximSchemes.contains(scheme)) { throw new SemanticException( ErrorMsg.INVALID_PATH .getMsg("only the following file systems accepted for export/import : " return new URI(scheme, authority, path, null, null); } catch (URISyntaxException e) { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); throw new SemanticException(ErrorMsg.IO_ERROR.getMsg() + ": " + e.getMessage(), e);
public static String relativeToAbsolutePath(HiveConf conf, String location) throws SemanticException { try { boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE) || conf.getBoolVar(HiveConf.ConfVars.HIVEEXIMTESTMODE);; if (testMode) { URI uri = new Path(location).toUri(); uri = new URI(scheme, authority, path, null, null); } catch (URISyntaxException e) { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); throw new SemanticException(ErrorMsg.IO_ERROR.getMsg() + ": " + e.getMessage(), e);
if (SessionState.get().isHiveServerQuery() && conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) { resFileFormat = "SequenceFile"; resultTab = ThriftJDBCBinarySerDe.class); } else { resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT); resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat, StatsTask columnStatsTask = map.get(tableName); if (columnStatsTask == null) { throw new SemanticException("Can not find " + tableName + " in genColumnStatsTask"); } else { columnStatsTask.getWork().setColStats(cStatsDesc);
DatabaseEventsIterator(Path dbLevelPath, HiveConf hiveConf) throws IOException { this.dbLevelPath = dbLevelPath; this.hiveConf = hiveConf; FileSystem fileSystem = dbLevelPath.getFileSystem(hiveConf); // this is only there for the use case where we are doing table only replication and not database level if (!fileSystem.exists(new Path(dbLevelPath + Path.SEPARATOR + EximUtil.METADATA_NAME))) { databaseEventProcessed = true; } if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_IN_REPL_TEST_FILES_SORTED)) { LOG.info(" file sorting is enabled in DatabaseEventsIterator"); List<LocatedFileStatus> fileStatuses = new ArrayList<>(); // Sort the directories in tha path and then add the files recursively . getSortedFileList(dbLevelPath, fileStatuses, fileSystem); remoteIterator = new RemoteIterator<LocatedFileStatus>() { private int idx = 0; private final int numEntry = fileStatuses.size(); private final List<LocatedFileStatus> fileStatusesLocal = fileStatuses; public boolean hasNext() throws IOException { return idx < numEntry; } public LocatedFileStatus next() throws IOException { LOG.info(" file in next is " + fileStatusesLocal.get(idx)); return fileStatusesLocal.get(idx++); } }; } else { remoteIterator = fileSystem.listFiles(dbLevelPath, true); } }