/** * Return partitioning columns for this input, can only be called after setInput is called, * since that takes care of adding a populated InputJobInfo object to its list in this job conf. * @return partitioning columns of the table specified by the job. * @throws IOException */ public static HCatSchema getPartitionColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = HCatUtil.getLastInputJobInfosFromConf(conf); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getPartitionColumns(); }
/** * Return data columns for this input, can only be called after setInput is called, * since that takes care of adding a populated InputJobInfo object to its list in this job conf. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = HCatUtil.getLastInputJobInfosFromConf(conf); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
private static PartInfo extractPartInfo(HCatSchema schema, StorageDescriptor sd, Map<String, String> parameters, Configuration conf, InputJobInfo inputJobInfo) throws IOException { StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd, parameters); Properties hcatProperties = new Properties(); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(conf, storerInfo); // copy the properties from storageHandler to jobProperties Map<String, String> jobProperties = HCatUtil.getInputJobProperties(storageHandler, inputJobInfo); for (String key : parameters.keySet()) { hcatProperties.put(key, parameters.get(key)); } // FIXME // Bloating partinfo with inputJobInfo is not good return new PartInfo(schema, storageHandler, sd.getLocation(), hcatProperties, jobProperties, inputJobInfo.getTableInfo()); }
HCatTableInfo tableInfo = inputJobInfo.getTableInfo(); HCatSchema dataColumns = tableInfo.getDataColumns(); List<HCatFieldSchema> dataFields = dataColumns.getFields();
/** * Gets the HCatTable schema for the table specified in the HCatInputFormat.setInput call * on the specified job context. This information is available only after HCatInputFormat.setInput * has been called for a JobContext. * @param conf the Configuration object * @return the table schema * @throws IOException if HCatInputFormat.setInput has not been called * for the current context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { InputJobInfo inputJobInfo = getJobInfo(conf); HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>()); for (HCatFieldSchema field : inputJobInfo.getTableInfo().getDataColumns().getFields()) { allCols.append(field); } for (HCatFieldSchema field : inputJobInfo.getTableInfo().getPartitionColumns().getFields()) { allCols.append(field); } return allCols; }
public static Map<String, String> getInputJobProperties(HiveStorageHandler storageHandler, InputJobInfo inputJobInfo) { Properties props = inputJobInfo.getTableInfo().getStorerInfo().getProperties(); props.put(serdeConstants.SERIALIZATION_LIB,storageHandler.getSerDeClass().getName()); TableDesc tableDesc = new TableDesc(storageHandler.getInputFormatClass(), storageHandler.getOutputFormatClass(),props); if (tableDesc.getJobProperties() == null) { tableDesc.setJobProperties(new HashMap<String, String>()); } Properties mytableProperties = tableDesc.getProperties(); mytableProperties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,inputJobInfo.getDatabaseName()+ "." + inputJobInfo.getTableName()); Map<String, String> jobProperties = new HashMap<String, String>(); try { Map<String, String> properties = tableDesc.getJobProperties(); LinkedList<InputJobInfo> inputJobInfos = (LinkedList<InputJobInfo>) HCatUtil.deserialize( properties.get(HCatConstants.HCAT_KEY_JOB_INFO)); if (inputJobInfos == null) { inputJobInfos = new LinkedList<>(); } inputJobInfos.add(inputJobInfo); properties.put(HCatConstants.HCAT_KEY_JOB_INFO, HCatUtil.serialize(inputJobInfos)); storageHandler.configureInputJobProperties(tableDesc, jobProperties); } catch (IOException e) { throw new IllegalStateException( "Failed to configure StorageHandler", e); } return jobProperties; }
inputJobInfo.getTableInfo());
HCatUtil.getLastInputJobInfosFromConf(taskContext.getConfiguration()).getTableInfo());
/** * Return partitioning columns for this input, can only be called after setInput is called. * @return partitioning columns of the table specified by the job. * @throws IOException */ public static HCatSchema getPartitionColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getPartitionColumns(); }
/** * Return partitioning columns for this input, can only be called after setInput is called. * @return partitioning columns of the table specified by the job. * @throws IOException */ public static HCatSchema getPartitionColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getPartitionColumns(); }
/** * Return partitioning columns for this input, can only be called after setInput is called. * @return partitioning columns of the table specified by the job. * @throws IOException */ public static HCatSchema getPartitionColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getPartitionColumns(); }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Return partitioning columns for this input, can only be called after setInput is called. * @return partitioning columns of the table specified by the job. * @throws IOException */ public static HCatSchema getPartitionColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getPartitionColumns(); }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
private static PartInfo extractPartInfo(HCatSchema schema, StorageDescriptor sd, Map<String, String> parameters, Configuration conf, InputJobInfo inputJobInfo) throws IOException { StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd, parameters); Properties hcatProperties = new Properties(); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(conf, storerInfo); // copy the properties from storageHandler to jobProperties Map<String, String> jobProperties = HCatUtil.getInputJobProperties(storageHandler, inputJobInfo); for (String key : parameters.keySet()) { hcatProperties.put(key, parameters.get(key)); } // FIXME // Bloating partinfo with inputJobInfo is not good return new PartInfo(schema, storageHandler, sd.getLocation(), hcatProperties, jobProperties, inputJobInfo.getTableInfo()); }
private static PartInfo extractPartInfo(HCatSchema schema, StorageDescriptor sd, Map<String, String> parameters, Configuration conf, InputJobInfo inputJobInfo) throws IOException { StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd, parameters); Properties hcatProperties = new Properties(); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(conf, storerInfo); // copy the properties from storageHandler to jobProperties Map<String, String> jobProperties = HCatUtil.getInputJobProperties(storageHandler, inputJobInfo); for (String key : parameters.keySet()) { hcatProperties.put(key, parameters.get(key)); } // FIXME // Bloating partinfo with inputJobInfo is not good return new PartInfo(schema, storageHandler, sd.getLocation(), hcatProperties, jobProperties, inputJobInfo.getTableInfo()); }
private static PartInfo extractPartInfo(HCatSchema schema, StorageDescriptor sd, Map<String, String> parameters, Configuration conf, InputJobInfo inputJobInfo) throws IOException { StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd, parameters); Properties hcatProperties = new Properties(); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(conf, storerInfo); // copy the properties from storageHandler to jobProperties Map<String, String> jobProperties = HCatUtil.getInputJobProperties(storageHandler, inputJobInfo); for (String key : parameters.keySet()) { hcatProperties.put(key, parameters.get(key)); } // FIXME // Bloating partinfo with inputJobInfo is not good return new PartInfo(schema, storageHandler, sd.getLocation(), hcatProperties, jobProperties, inputJobInfo.getTableInfo()); }
private static PartInfo extractPartInfo(HCatSchema schema, StorageDescriptor sd, Map<String, String> parameters, Configuration conf, InputJobInfo inputJobInfo) throws IOException { StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd, parameters); Properties hcatProperties = new Properties(); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(conf, storerInfo); // copy the properties from storageHandler to jobProperties Map<String, String> jobProperties = HCatUtil.getInputJobProperties(storageHandler, inputJobInfo); for (String key : parameters.keySet()) { hcatProperties.put(key, parameters.get(key)); } // FIXME // Bloating partinfo with inputJobInfo is not good return new PartInfo(schema, storageHandler, sd.getLocation(), hcatProperties, jobProperties, inputJobInfo.getTableInfo()); }