if (!rhs.getCols().containsAll(getCols()) || !getCols().containsAll(rhs.getCols())) { theDiff.add(TableAttribute.COLUMNS); if ((getInputFileFormat() == null && rhs.getInputFileFormat() != null) || (getInputFileFormat() != null && (rhs.getInputFileFormat() == null || !rhs.getInputFileFormat().equals(getInputFileFormat())))) { theDiff.add(TableAttribute.INPUT_FORMAT); if ((getOutputFileFormat() == null && rhs.getOutputFileFormat() != null) || (getOutputFileFormat() != null && (rhs.getOutputFileFormat() == null || !rhs.getOutputFileFormat().equals(getOutputFileFormat())))) { theDiff.add(TableAttribute.OUTPUT_FORMAT); if ((getStorageHandler() == null && rhs.getStorageHandler() != null) || (getStorageHandler() != null && (rhs.getStorageHandler() == null || !rhs.getStorageHandler().equals(getStorageHandler())))) { theDiff.add(TableAttribute.STORAGE_HANDLER); if ((getSerdeLib() == null && rhs.getSerdeLib() != null) || (getSerdeLib() != null && (rhs.getSerdeLib() == null || !rhs.getSerdeLib().equals(getSerdeLib())))) { theDiff.add(TableAttribute.SERDE); if (!equivalent(sd.getSerdeInfo().getParameters(), rhs.sd.getSerdeInfo().getParameters())) { theDiff.add(TableAttribute.SERDE_PROPERTIES); if (!equivalent(tblProps, rhs.tblProps)) { theDiff.add(TableAttribute.TABLE_PROPERTIES);
/** * Setter for partition key-values. */ public HCatPartition setPartitionKeyValues(Map<String, String> partitionKeyValues) throws HCatException { for (HCatFieldSchema partField : hcatTable.getPartCols()) { if (!partitionKeyValues.containsKey(partField.getName())) { throw new HCatException("Missing value for partition-key \'" + partField.getName() + "\' in table: " + hcatTable.getDbName() + "." + hcatTable.getTableName()); } else { values.add(partitionKeyValues.get(partField.getName())); // Keep partKeyValMap in synch as well. } } return this; }
@Override public String toString() { return "HCatTable [ " + "tableName=" + tableName + ", " + "dbName=" + dbName + ", " + "tableType=" + tableType + ", " + "cols=" + cols + ", " + "partCols=" + partCols + ", " + "bucketCols=" + getBucketCols() + ", " + "numBuckets=" + getNumBuckets() + ", " + "sortCols=" + getSortCols() + ", " + "inputFormat=" + getInputFileFormat() + ", " + "outputFormat=" + getOutputFileFormat() + ", " + "storageHandler=" + getStorageHandler() + ", " + "serde=" + getSerdeLib() + ", " + "tblProps=" + getTblProps() + ", " + "location=" + getLocation() + ", " + "owner=" + owner() + " ]"; }
cols(rhs.cols); inputFileFormat(rhs.getInputFileFormat()); outputFileFormat(rhs.getOutputFileFormat()); serdeLib(rhs.getSerdeLib()); serdeParams(rhs.getSerdeParams()); storageHandler(rhs.getStorageHandler()); tblProps(rhs.tblProps);
new HCatFieldSchema("grid", Type.STRING, "")); HCatTable sourceTable = new HCatTable(dbName, tableName).cols(columnSchema) .partCols(partitionSchema) .comment("Source table."); assertEquals("Column schema doesn't match.", sourceTable.getCols(), addedPartition_1.getColumns()); assertEquals("InputFormat doesn't match.", sourceTable.getInputFileFormat(), addedPartition_1.getInputFormat()); assertEquals("OutputFormat doesn't match.", sourceTable.getOutputFileFormat(), addedPartition_1.getOutputFormat()); assertEquals("SerDe doesn't match.", sourceTable.getSerdeLib(), addedPartition_1.getSerDe()); assertEquals("SerDe params don't match.", sourceTable.getSerdeParams(), addedPartition_1.getSerdeParams()); targetTable = targetMetaStore().getTable(dbName, tableName); assertEquals("Created table doesn't match the source.", HCatTable.NO_DIFF, targetTable.diff(sourceTable)); Map<String, String> tableParams = new HashMap<String, String>(1); tableParams.put("orc.compress", "ZLIB"); sourceTable.cols(newColumnSchema) // Add a column. .fileFormat("orcfile") // Change SerDe, File I/O formats. .tblProps(tableParams) .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001')); sourceMetaStore().updateTableSchema(dbName, tableName, sourceTable); sourceTable = sourceMetaStore().getTable(dbName, tableName);
new HCatFieldSchema("grid", Type.STRING, "")); HCatTable sourceTable = new HCatTable(dbName, tableName).cols(columnSchema).partCols(partitionSchema); sourceMetaStore().createTable(HCatCreateTableDesc.create(sourceTable).build()); HCatTable.NO_DIFF, sourceTable.diff(targetTable)); HCatTable.NO_DIFF, sourceTable.diff(targetTable)); Map<String, String> tableParams = new HashMap<String, String>(1); tableParams.put("orc.compress", "ZLIB"); sourceTable.cols(newColumnSchema) // Add a column. .fileFormat("orcfile") // Change SerDe, File I/O formats. .tblProps(tableParams) .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001')); sourceMetaStore().updateTableSchema(dbName, tableName, sourceTable); sourceTable = sourceMetaStore().getTable(dbName, tableName); EnumSet<HCatTable.TableAttribute> diff = targetTable.diff(sourceTable); assertTrue("Couldn't find change in column-schema.", diff.contains(HCatTable.TableAttribute.COLUMNS)); targetMetaStore().updateTableSchema(dbName, tableName, targetTable.resolve(sourceTable, diff)); targetTable = targetMetaStore().getTable(dbName, tableName); HCatTable.NO_DIFF, targetTable.diff(sourceTable));
client.createTable(tableDesc); HCatTable table1 = client.getTable(db, tableOne); assertTrue(table1.getInputFileFormat().equalsIgnoreCase( RCFileInputFormat.class.getName())); assertTrue(table1.getOutputFileFormat().equalsIgnoreCase( RCFileOutputFormat.class.getName())); assertTrue(table1.getSerdeLib().equalsIgnoreCase( LazyBinaryColumnarSerDe.class.getName())); assertTrue(table1.getCols().equals(cols)); client.createTable(tableDesc2); HCatTable table2 = client.getTable(db, tableTwo); assertTrue("Expected TextInputFormat, but got: " + table2.getInputFileFormat(), table2.getInputFileFormat().equalsIgnoreCase(TextInputFormat.class.getName())); assertTrue(table2.getOutputFileFormat().equalsIgnoreCase( HiveIgnoreKeyTextOutputFormat.class.getName())); assertTrue("SerdeParams not found", table2.getSerdeParams() != null); assertEquals("checking " + serdeConstants.FIELD_DELIM, Character.toString('\001'), table2.getSerdeParams().get(serdeConstants.FIELD_DELIM)); assertEquals("checking " + serdeConstants.ESCAPE_CHAR, Character.toString('\002'), table2.getSerdeParams().get(serdeConstants.ESCAPE_CHAR)); assertEquals("checking " + serdeConstants.LINE_DELIM, Character.toString('\003'), table2.getSerdeParams().get(serdeConstants.LINE_DELIM)); assertEquals("checking " + serdeConstants.MAPKEY_DELIM, Character.toString('\004'), table2.getSerdeParams().get(serdeConstants.MAPKEY_DELIM)); assertEquals("checking " + serdeConstants.COLLECTION_DELIM, Character.toString('\005'), table2.getSerdeParams().get(serdeConstants.COLLECTION_DELIM)); assertEquals("checking " + serdeConstants.SERIALIZATION_NULL_FORMAT, Character.toString('\006'), table2.getSerdeParams().get(serdeConstants.SERIALIZATION_NULL_FORMAT));
ArrayList<HCatFieldSchema> partitionColumns = new ArrayList<HCatFieldSchema>( Arrays.asList(new HCatFieldSchema(partitionColumn, Type.STRING, ""))); HCatTable table = new HCatTable(dbName, tableName).cols(columns).partCols(partitionColumns); client.createTable(HCatCreateTableDesc.create(table, false).build());
public HCatPartition(HCatTable hcatTable, Map<String, String> partitionKeyValues, String location) throws HCatException { this.hcatTable = hcatTable; this.tableName = hcatTable.getTableName(); this.dbName = hcatTable.getDbName(); this.sd = new StorageDescriptor(hcatTable.getSd()); this.sd.setLocation(location); this.columns = getColumns(this.sd); this.createTime = (int)(System.currentTimeMillis()/1000); this.lastAccessTime = -1; this.values = new ArrayList<String>(hcatTable.getPartCols().size()); for (HCatFieldSchema partField : hcatTable.getPartCols()) { if (!partitionKeyValues.containsKey(partField.getName())) { throw new HCatException("Missing value for partition-key \'" + partField.getName() + "\' in table: " + hcatTable.getDbName() + "." + hcatTable.getTableName()); } else { values.add(partitionKeyValues.get(partField.getName())); } } }
client.createDatabase(HCatCreateDBDesc.create(dbName).ifNotExists(false).build()); HCatTable table = (new HCatTable(dbName, tableName)).cols(cols).fileFormat("textfile"); assertNotNull(importedTable); assertTrue(importedTable.getTblProps().containsKey("repl.last.id"));
/** * This test tests that a plain table instantiation matches what hive says an * empty table create should look like. * @throws Exception */ @Test public void testEmptyTableInstantiation() throws Exception { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String dbName = "default"; String tblName = "testEmptyCreate"; ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); cols.add(new HCatFieldSchema("value", Type.STRING, "value comment")); client.dropTable(dbName, tblName, true); // Create a minimalistic table client.createTable(HCatCreateTableDesc .create(new HCatTable(dbName, tblName).cols(cols), false) .build()); HCatTable tCreated = client.getTable(dbName, tblName); org.apache.hadoop.hive.metastore.api.Table emptyTable = Table.getEmptyTable(dbName, tblName); Map<String, String> createdProps = tCreated.getTblProps(); Map<String, String> emptyProps = emptyTable.getParameters(); mapEqualsContainedIn(emptyProps, createdProps); // Test sd params - we check that all the parameters in an empty table // are retained as-is. We may add beyond it, but not change values for // any parameters that hive defines for an empty table. Map<String, String> createdSdParams = tCreated.getSerdeParams(); Map<String, String> emptySdParams = emptyTable.getSd().getSerdeInfo().getParameters(); mapEqualsContainedIn(emptySdParams, createdSdParams); }
ptnCols.add(new HCatFieldSchema("country", Type.STRING, "country column")); HCatTable table = new HCatTable(dbName, tableName).cols(cols) .partCols(ptnCols) .fileFormat("sequenceFile"); HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(table, false).build(); client.createTable(tableDesc);
/** * Gets the partition columns of the table. * * @return the partition columns */ public List<HCatFieldSchema> getPartColumns() { return hcatTable.getPartCols(); }
TableDataBuilder(HCatTable table) { schema = new HCatSchema(ImmutableList .<HCatFieldSchema> builder() .addAll(table.getCols()) .addAll(table.getPartCols()) .build()); partitionColumns = table.getPartCols(); withAllColumns(); }
@Override public HCatTable deserializeTable(String hcatTableStringRep) throws HCatException { try { Table table = new Table(); new TDeserializer(new TJSONProtocol.Factory()).deserialize(table, hcatTableStringRep, "UTF-8"); return new HCatTable(table); } catch(TException exception) { if (LOG.isDebugEnabled()) LOG.debug("Could not de-serialize from: " + hcatTableStringRep); throw new HCatException("Could not de-serialize HCatTable.", exception); } }
/** * Gets the file format. * * @return the file format */ @Deprecated // @deprecated in favour of {@link HCatTable.#fileFormat()}. To be removed in Hive 0.16. public String getFileFormat() { return this.hcatTable.fileFormat(); }
/** * Gets the comments. * * @return the comments */ @Deprecated // @deprecated in favour of {@link HCatTable.#comment()}. To be removed in Hive 0.16. public String getComments() { return this.hcatTable.comment(); }