@Override public void configureOutputJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) { jobProperties.put(Constants.DRUID_DATA_SOURCE, tableDesc.getTableName()); jobProperties.put(DruidConstants.DRUID_SEGMENT_VERSION, new DateTime().toString()); jobProperties.put(DruidConstants.DRUID_JOB_WORKING_DIRECTORY, getStagingWorkingDir().toString()); // DruidOutputFormat will write segments in an intermediate directory jobProperties.put(DruidConstants.DRUID_SEGMENT_INTERMEDIATE_DIRECTORY, getIntermediateSegmentDir().toString()); }
protected ValidWriteIdList getMmValidWriteIds( JobConf conf, TableDesc table, ValidWriteIdList validWriteIdList) throws IOException { if (!AcidUtils.isInsertOnlyTable(table.getProperties())) return null; if (validWriteIdList == null) { validWriteIdList = AcidUtils.getTableValidWriteIdList( conf, table.getTableName()); if (validWriteIdList == null) { throw new IOException("Insert-Only table: " + table.getTableName() + " is missing from the ValidWriteIdList config: " + conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); } } return validWriteIdList; }
public LoadTableDesc(final Path sourcePath, final TableDesc table, final Map<String, String> partitionSpec, final LoadFileType loadFileType, final AcidUtils.Operation writeType, Long currentWriteId) { super(sourcePath, writeType); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("creating part LTD from " + sourcePath + " to " + ((table.getProperties() == null) ? "null" : table.getTableName())); } init(table, partitionSpec, loadFileType, currentWriteId); }
public String getCounterName(Counter counter) { String suffix = Integer.toString(conf.getDestTableId()); String fullName = conf.getTableInfo().getTableName(); if (fullName != null) { suffix = suffix + "_" + fullName.toLowerCase(); } return counter + "_" + suffix; }
public String getCounterName(Counter counter) { String suffix = Integer.toString(conf.getDestTableId()); String fullName = conf.getTableInfo().getTableName(); if (fullName != null) { suffix = suffix + "_" + fullName.toLowerCase(); } return counter + "_" + suffix; }
/** * Returns the table location path from a TableDesc object. * * @param hconf Configuration object. * @param tableDesc Table description from where to get the table name. * @return The path where the table is located. */ private static Path getTableLocationPath(final HiveConf hconf, final TableDesc tableDesc) { Table table = null; try { Hive hive = Hive.get(hconf); table = hive.getTable(tableDesc.getTableName()); } catch (HiveException e) { LOG.warn("Unable to get the table location path for: " + tableDesc.getTableName(), e); } return (table != null) ? table.getPath() : null; }
public void logMessage(LoadTableDesc tbd) { StringBuilder mesg = new StringBuilder("Loading data to table ") .append( tbd.getTable().getTableName()); if (tbd.getPartitionSpec().size() > 0) { mesg.append(" partition ("); Map<String, String> partSpec = tbd.getPartitionSpec(); for (String key: partSpec.keySet()) { mesg.append(key).append('=').append(partSpec.get(key)).append(", "); } mesg.setLength(mesg.length()-2); mesg.append(')'); } String mesg_detail = " from " + tbd.getSourcePath(); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace(mesg.toString() + " " + mesg_detail); } console.printInfo(mesg.toString(), mesg_detail); }
/** * Returns true iff current query is an insert into for the given file sink * * @param parseCtx * @param fsOp * @return */ public static boolean isInsertInto(ParseContext parseCtx, FileSinkOperator fsOp) { return fsOp.getConf().getTableInfo().getTableName() != null; }
/** * Returns the table location path from a TableDesc object. * * @param hconf Configuration object. * @param tableDesc Table description from where to get the table name. * @return The path where the table is located. */ private static Path getTableLocationPath(final HiveConf hconf, final TableDesc tableDesc) { Table table = null; try { Hive hive = Hive.get(hconf); table = hive.getTable(tableDesc.getTableName()); } catch (HiveException e) { LOG.warn("Unable to get the table location path for: " + tableDesc.getTableName(), e); } return (table != null) ? table.getPath() : null; }
/** * Copy job credentials to table properties * @param tbl */ public static void copyJobSecretToTableProperties(TableDesc tbl) throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); for (Text key : credentials.getAllSecretKeys()) { String keyString = key.toString(); if (keyString.startsWith(TableDesc.SECRET_PREFIX + TableDesc.SECRET_DELIMIT)) { String[] comps = keyString.split(TableDesc.SECRET_DELIMIT); String tblName = comps[1]; String keyName = comps[2]; if (tbl.getTableName().equalsIgnoreCase(tblName)) { tbl.getProperties().put(keyName, new String(credentials.getSecretKey(key))); } } } }
/** * Returns true iff current query is an insert into for the given file sink * * @param parseCtx * @param fsOp * @return */ public static boolean isInsertInto(ParseContext parseCtx, FileSinkOperator fsOp) { return fsOp.getConf().getTableInfo().getTableName() != null; }
public LoadTableDesc(final Path sourcePath, final TableDesc table, final DynamicPartitionCtx dpCtx, final AcidUtils.Operation writeType, boolean isReplace, Long writeId) { super(sourcePath, writeType); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("creating LTD from " + sourcePath + " to " + table.getTableName()); } this.dpCtx = dpCtx; LoadFileType lft = isReplace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING; if (dpCtx != null && dpCtx.getPartSpec() != null && partitionSpec == null) { init(table, dpCtx.getPartSpec(), lft, writeId); } else { init(table, new LinkedHashMap<String, String>(), lft, writeId); } }
public static void configureJobConf(TableDesc tableDesc, JobConf jobConf) { String handlerClass = tableDesc.getProperties().getProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); try { HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(jobConf, handlerClass); if (storageHandler != null) { storageHandler.configureJobConf(tableDesc, jobConf); } if (tableDesc.getJobSecrets() != null) { for (Map.Entry<String, String> entry : tableDesc.getJobSecrets().entrySet()) { String key = TableDesc.SECRET_PREFIX + TableDesc.SECRET_DELIMIT + tableDesc.getTableName() + TableDesc.SECRET_DELIMIT + entry.getKey(); jobConf.getCredentials().addSecretKey(new Text(key), entry.getValue().getBytes()); } tableDesc.getJobSecrets().clear(); } } catch (HiveException e) { throw new RuntimeException(e); } }
private void genAutoColumnStatsGatheringPipeline(QB qb, TableDesc table_desc, Map<String, String> partSpec, Operator curr, boolean isInsertInto) throws SemanticException { String tableName = table_desc.getTableName(); Table table = null; try { table = db.getTable(tableName); } catch (HiveException e) { throw new SemanticException(e.getMessage()); } LOG.info("Generate an operator pipeline to autogather column stats for table " + tableName + " in query " + ctx.getCmd()); ColumnStatsAutoGatherContext columnStatsAutoGatherContext = null; columnStatsAutoGatherContext = new ColumnStatsAutoGatherContext(this, conf, curr, table, partSpec, isInsertInto, ctx); columnStatsAutoGatherContext.insertAnalyzePipeline(); columnStatsAutoGatherContexts.add(columnStatsAutoGatherContext); }
public List<Task<? extends Serializable>> generateUpdateTasks() throws HiveException { hive = Hive.get(this.conf); for (LoadTableDesc ltd : loadTableWork) { TableDesc td = ltd.getTable(); Table srcTable = hive.getTable(td.getTableName()); List<Index> tblIndexes = IndexUtils.getAllIndexes(srcTable, (short)-1); Map<String, String> partSpec = ltd.getPartitionSpec(); if (partSpec == null || partSpec.size() == 0) { //unpartitioned table, update whole index doIndexUpdate(tblIndexes); } else { doIndexUpdate(tblIndexes, partSpec); } } return tasks; }
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
private void checkAcidConstraints(QB qb, TableDesc tableDesc, Table table) throws SemanticException { String tableName = tableDesc.getTableName(); if (!qb.getParseInfo().isInsertIntoTable(tableName)) { LOG.debug("Couldn't find table " + tableName + " in insertIntoTable"); throw new SemanticException(ErrorMsg.NO_INSERT_OVERWRITE_WITH_ACID.getMsg()); } /* LOG.info("Modifying config values for ACID write"); conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, true); conf.setIntVar(ConfVars.HIVEOPTREDUCEDEDUPLICATIONMINREDUCER, 1); These props are now enabled elsewhere (see commit diffs). It would be better instead to throw if they are not set. For exmaple, if user has set hive.optimize.reducededuplication=false for some reason, we'll run a query contrary to what they wanted... But throwing now would be backwards incompatible. */ conf.set(AcidUtils.CONF_ACID_KEY, "true"); if (table.getNumBuckets() < 1) { throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName()); } if (table.getSortCols() != null && table.getSortCols().size() > 0) { throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName()); } }
public String getTableName() { BasicStatsWork work = this; if (work.getLoadTableDesc() != null) { return work.getLoadTableDesc().getTable().getTableName(); } else if (work.getTableSpecs() != null) { return work.getTableSpecs().tableName; } else if (getLoadFileDesc().getCtasCreateTableDesc() != null) { return getLoadFileDesc().getCtasCreateTableDesc().getTableName(); } else { return getLoadFileDesc().getCreateViewDesc().getViewName(); } }
private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() + " into " + tbd.getTable().getTableName()); } db.loadPartition(tbd.getSourcePath(), db.getTable(tbd.getTable().getTableName()), tbd.getPartitionSpec(), tbd.getLoadFileType(), tbd.getInheritTableSpecs(), tbd.getInheritLocation(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && !tbd.isMmTable(), resetStatisticsProps(table), tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); // See the comment inside updatePartitionBucketSortColumns. if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) { updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols); } DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition()); // add this partition to post-execution hook if (work.getOutputs() != null) { DDLTask.addIfAbsentByName(new WriteEntity(partn, getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs()); } return dc; }