/** * Return data columns for this input, can only be called after setInput is called, * since that takes care of adding a populated InputJobInfo object to its list in this job conf. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = HCatUtil.getLastInputJobInfosFromConf(conf); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Gets the table schema for the table specified in the HCatOutputFormat.setOutput call * on the specified job context. * Note: This is the record-schema for the table. It does not include the table's partition columns. * @param conf the Configuration object * @return the table schema, excluding partition columns * @throws IOException if HCatOutputFormat.setOutput has not been called for the passed context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { OutputJobInfo jobInfo = getJobInfo(conf); return jobInfo.getTableInfo().getDataColumns(); }
HCatSchema dataColumns = tableInfo.getDataColumns(); List<HCatFieldSchema> dataFields = dataColumns.getFields(); StringBuilder columnNamesSb = new StringBuilder();
assert tableInfo != null : "TableInfo can't be null at this point."; if (partitionSchema != null) { if (partitionSchema.equals(tableInfo.getDataColumns())) { partitionSchema = null; } else { LOG.debug("Can't suppress data-schema. Partition-schema and table-schema seem to differ! " + " partitionSchema: " + partitionSchema.getFields() + " tableSchema: " + tableInfo.getDataColumns());
/** * Gets the HCatTable schema for the table specified in the HCatInputFormat.setInput call * on the specified job context. This information is available only after HCatInputFormat.setInput * has been called for a JobContext. * @param conf the Configuration object * @return the table schema * @throws IOException if HCatInputFormat.setInput has not been called * for the current context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { InputJobInfo inputJobInfo = getJobInfo(conf); HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>()); for (HCatFieldSchema field : inputJobInfo.getTableInfo().getDataColumns().getFields()) { allCols.append(field); } for (HCatFieldSchema field : inputJobInfo.getTableInfo().getPartitionColumns().getFields()) { allCols.append(field); } return allCols; }
public void testSetOutput() throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "test outputformat"); Map<String, String> partitionValues = new HashMap<String, String>(); partitionValues.put("colname", "p1"); //null server url means local mode OutputJobInfo info = OutputJobInfo.create(dbName, tblName, partitionValues); HCatOutputFormat.setOutput(job, info); OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(job.getConfiguration()); assertNotNull(jobInfo.getTableInfo()); assertEquals(1, jobInfo.getPartitionValues().size()); assertEquals("p1", jobInfo.getPartitionValues().get("colname")); assertEquals(1, jobInfo.getTableInfo().getDataColumns().getFields().size()); assertEquals("data_column", jobInfo.getTableInfo().getDataColumns().getFields().get(0).getName()); publishTest(job); }
/** * Undoes the effects of compression( dedupWithTableInfo() ) during serialization, * and restores PartInfo fields to return original data. * Can be called idempotently, repeatably. */ private void restoreLocalInfoFromTableInfo() { assert tableInfo != null : "TableInfo can't be null at this point."; if (partitionSchema == null) { partitionSchema = tableInfo.getDataColumns(); } if (storageHandlerClassName == null) { storageHandlerClassName = tableInfo.getStorerInfo().getStorageHandlerClass(); } if (inputFormatClassName == null) { inputFormatClassName = tableInfo.getStorerInfo().getIfClass(); } if (outputFormatClassName == null) { outputFormatClassName = tableInfo.getStorerInfo().getOfClass(); } if (serdeClassName == null) { serdeClassName = tableInfo.getStorerInfo().getSerdeClass(); } }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Gets the table schema for the table specified in the HCatOutputFormat.setOutput call * on the specified job context. * Note: This is the record-schema for the table. It does not include the table's partition columns. * @param conf the Configuration object * @return the table schema, excluding partition columns * @throws IOException if HCatOutputFormat.setOutput has not been called for the passed context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { OutputJobInfo jobInfo = getJobInfo(conf); return jobInfo.getTableInfo().getDataColumns(); }
/** * Gets the table schema for the table specified in the HCatOutputFormat.setOutput call * on the specified job context. * @param conf the Configuration object * @return the table schema * @throws IOException if HCatOutputFormat.setOutput has not been called for the passed context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { OutputJobInfo jobInfo = getJobInfo(conf); return jobInfo.getTableInfo().getDataColumns(); }
/** * Gets the table schema for the table specified in the HCatOutputFormat.setOutput call * on the specified job context. * @param conf the Configuration object * @return the table schema * @throws IOException if HCatOutputFormat.setOutput has not been called for the passed context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { OutputJobInfo jobInfo = getJobInfo(conf); return jobInfo.getTableInfo().getDataColumns(); }
/** * Gets the table schema for the table specified in the HCatOutputFormat.setOutput call * on the specified job context. * @param conf the Configuration object * @return the table schema * @throws IOException if HCatOutputFormat.setOutput has not been called for the passed context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { OutputJobInfo jobInfo = getJobInfo(conf); return jobInfo.getTableInfo().getDataColumns(); }
/** * Return data columns for this input, can only be called after setInput is called. * @return data columns of the table specified by the job. * @throws IOException */ public static HCatSchema getDataColumns(Configuration conf) throws IOException { InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize( conf.get(HCatConstants.HCAT_KEY_JOB_INFO)); Preconditions.checkNotNull(inputInfo, "inputJobInfo is null, setInput has not yet been called to save job into conf supplied."); return inputInfo.getTableInfo().getDataColumns(); } }
/** * Gets the HCatTable schema for the table specified in the HCatInputFormat.setInput call * on the specified job context. This information is available only after HCatInputFormat.setInput * has been called for a JobContext. * @param conf the Configuration object * @return the table schema * @throws IOException if HCatInputFormat.setInput has not been called * for the current context */ public static HCatSchema getTableSchema(Configuration conf) throws IOException { InputJobInfo inputJobInfo = getJobInfo(conf); HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>()); for (HCatFieldSchema field : inputJobInfo.getTableInfo().getDataColumns().getFields()) { allCols.append(field); } for (HCatFieldSchema field : inputJobInfo.getTableInfo().getPartitionColumns().getFields()) { allCols.append(field); } return allCols; }
void setTableInfo(HCatTableInfo thatTableInfo) { this.tableInfo = thatTableInfo; if (partitionSchema == null) { partitionSchema = tableInfo.getDataColumns(); } if (storageHandlerClassName == null) { storageHandlerClassName = tableInfo.getStorerInfo().getStorageHandlerClass(); } if (inputFormatClassName == null) { inputFormatClassName = tableInfo.getStorerInfo().getIfClass(); } if (outputFormatClassName == null) { outputFormatClassName = tableInfo.getStorerInfo().getOfClass(); } if (serdeClassName == null) { serdeClassName = tableInfo.getStorerInfo().getSerdeClass(); } }
void setTableInfo(HCatTableInfo thatTableInfo) { this.tableInfo = thatTableInfo; if (partitionSchema == null) { partitionSchema = tableInfo.getDataColumns(); } if (storageHandlerClassName == null) { storageHandlerClassName = tableInfo.getStorerInfo().getStorageHandlerClass(); } if (inputFormatClassName == null) { inputFormatClassName = tableInfo.getStorerInfo().getIfClass(); } if (outputFormatClassName == null) { outputFormatClassName = tableInfo.getStorerInfo().getOfClass(); } if (serdeClassName == null) { serdeClassName = tableInfo.getStorerInfo().getSerdeClass(); } }
/** * Undoes the effects of compression( dedupWithTableInfo() ) during serialization, * and restores PartInfo fields to return original data. * Can be called idempotently, repeatably. */ private void restoreLocalInfoFromTableInfo() { assert tableInfo != null : "TableInfo can't be null at this point."; if (partitionSchema == null) { partitionSchema = tableInfo.getDataColumns(); } if (storageHandlerClassName == null) { storageHandlerClassName = tableInfo.getStorerInfo().getStorageHandlerClass(); } if (inputFormatClassName == null) { inputFormatClassName = tableInfo.getStorerInfo().getIfClass(); } if (outputFormatClassName == null) { outputFormatClassName = tableInfo.getStorerInfo().getOfClass(); } if (serdeClassName == null) { serdeClassName = tableInfo.getStorerInfo().getSerdeClass(); } }
/** * Undoes the effects of compression( dedupWithTableInfo() ) during serialization, * and restores PartInfo fields to return original data. * Can be called idempotently, repeatably. */ private void restoreLocalInfoFromTableInfo() { assert tableInfo != null : "TableInfo can't be null at this point."; if (partitionSchema == null) { partitionSchema = tableInfo.getDataColumns(); } if (storageHandlerClassName == null) { storageHandlerClassName = tableInfo.getStorerInfo().getStorageHandlerClass(); } if (inputFormatClassName == null) { inputFormatClassName = tableInfo.getStorerInfo().getIfClass(); } if (outputFormatClassName == null) { outputFormatClassName = tableInfo.getStorerInfo().getOfClass(); } if (serdeClassName == null) { serdeClassName = tableInfo.getStorerInfo().getSerdeClass(); } }