private void alterPartitionSpecInMemory(Table tbl, Map<String, String> partSpec, org.apache.hadoop.hive.metastore.api.Partition tpart, boolean inheritTableSpecs, String partPath) throws HiveException, InvalidOperationException { LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); if (inheritTableSpecs) { tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); tpart.getSd().getSerdeInfo().setParameters( tbl.getTTable().getSd().getSerdeInfo().getParameters()); tpart.getSd().setBucketCols(tbl.getBucketCols()); tpart.getSd().setNumBuckets(tbl.getNumBuckets()); tpart.getSd().setSortCols(tbl.getSortCols()); } if (partPath == null || partPath.trim().equals("")) { throw new HiveException("new partition path should not be null or empty."); } tpart.getSd().setLocation(partPath); }
public static void fromMetastoreApiStorageDescriptor(StorageDescriptor storageDescriptor, Storage.Builder builder, String tablePartitionName) { SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo(); if (serdeInfo == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table storage descriptor is missing SerDe info"); } builder.setStorageFormat(StorageFormat.createNullable(serdeInfo.getSerializationLib(), storageDescriptor.getInputFormat(), storageDescriptor.getOutputFormat())) .setLocation(nullToEmpty(storageDescriptor.getLocation())) .setBucketProperty(HiveBucketProperty.fromStorageDescriptor(storageDescriptor, tablePartitionName)) .setSkewed(storageDescriptor.isSetSkewedInfo() && storageDescriptor.getSkewedInfo().isSetSkewedColNames() && !storageDescriptor.getSkewedInfo().getSkewedColNames().isEmpty()) .setSerdeParameters(serdeInfo.getParameters() == null ? ImmutableMap.of() : serdeInfo.getParameters()); }
Table build() { StorageDescriptor sd = new StorageDescriptor(); if (columns == null) { sd.setCols(Collections.emptyList()); } else { sd.setCols(columns); } SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setSerializationLib(serde); serdeInfo.setName(tableName); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(inputFormat); sd.setOutputFormat(outputFormat); if (location != null) { sd.setLocation(location); } Table table = new Table(); table.setDbName(dbName); table.setTableName(tableName); table.setSd(sd); table.setParameters(parameters); table.setOwner(owner); if (partitionKeys != null) { table.setPartitionKeys(partitionKeys); } table.setTableType(tableType.toString()); return table; } }
protected SerDeInfo buildSerde() { SerDeInfo serDeInfo = new SerDeInfo(serdeName, serdeLib, serdeParams); if (serdeDescription != null) serDeInfo.setDescription(serdeDescription); if (serdeSerializerClass != null) serDeInfo.setSerializerClass(serdeSerializerClass); if (serdeDeserializerClass != null) serDeInfo.setDeserializerClass(serdeDeserializerClass); if (serdeType != null) serDeInfo.setSerdeType(serdeType); return serDeInfo; }
public SerDeInfoWrapper(SerDeInfo serDeInfo) { this.serDeInfo = serDeInfo; this.name = serDeInfo.getName(); this.serializationLib = serDeInfo.getSerializationLib(); this.parameters = serDeInfo.getParameters(); }
public Object getFieldValue(_Fields field) { switch (field) { case NAME: return getName(); case SERIALIZATION_LIB: return getSerializationLib(); case PARAMETERS: return getParameters(); case DESCRIPTION: return getDescription(); case SERIALIZER_CLASS: return getSerializerClass(); case DESERIALIZER_CLASS: return getDeserializerClass(); case SERDE_TYPE: return getSerdeType(); } throw new IllegalStateException(); }
private static void createTable(String tableName, String tablePerm) throws Exception { Table tbl = new Table(); tbl.setDbName(DATABASE); tbl.setTableName(tableName); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(ColumnHolder.colMapping.get(tableName)); tbl.setSd(sd); sd.setParameters(new HashMap<String, String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib( org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); tbl.setPartitionKeys(ColumnHolder.partitionCols); hmsc.createTable(tbl); Path path = new Path(warehousedir, tableName); FileSystem fs = path.getFileSystem(hiveConf); fs.setPermission(path, new FsPermission(tablePerm)); }
@Test public void testCreateTableDefaultValues() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(DEFAULT_DATABASE); table.setTableName("test_table_2"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); Assert.assertEquals("Storage descriptor cols", 1, createdSd.getCols().size()); Assert.assertNull("Storage descriptor cols[0].comment", createdSd.getCols().get(0).getComment()); Assert.assertEquals("Storage descriptor location", metaStore.getWarehouseRoot() + "/" + table.getTableName(), createdSd.getLocation()); SerDeInfo serDeInfo = createdSd.getSerdeInfo(); Assert.assertNull("SerDeInfo name", serDeInfo.getName()); Assert.assertNull("SerDeInfo serialization lib", serDeInfo.getSerializationLib()); Assert.assertEquals("SerDeInfo parameters", 0, serDeInfo.getParameters().size());
private void verifyPartitionSharedSD(Table table, String name, List<String> values, int index) throws Exception { Partition part = client.getPartition(table.getDbName(), table.getTableName(), name); Assert.assertNotNull(part); Assert.assertEquals(table.getTableName(), part.getTableName()); List<String> partValues = part.getValues(); Assert.assertEquals(values.size(), partValues.size()); Assert.assertTrue(partValues.containsAll(values)); Assert.assertEquals(table.getDbName(), part.getDbName()); Assert.assertEquals(DEFAULT_CREATE_TIME, part.getLastAccessTime()); Assert.assertEquals(DEFAULT_PARAM_VALUE + index, part.getParameters().get(DEFAULT_PARAM_KEY + index)); Assert.assertFalse(part.getParameters().keySet().contains(table.getParameters().keySet())); StorageDescriptor sd = part.getSd(); Assert.assertNotNull(sd); Assert.assertEquals("TestInputFormat", sd.getInputFormat()); Assert.assertEquals("TestOutputFormat", sd.getOutputFormat()); Assert.assertEquals("sharedSDPartSerde", sd.getSerdeInfo().getName()); Assert.assertEquals("testSDParamValue", sd.getParameters().get("testSDParamKey")); Assert.assertEquals( metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/partwithoutsd" + index, sd.getLocation()); Assert.assertTrue(metaStore.isPathExists(new Path(sd.getLocation()))); }
StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo()); sd.setNumBuckets(-1); sd.setBucketCols(new ArrayList<String>()); sd.setCols(new ArrayList<FieldSchema>()); sd.setParameters(new HashMap<String, String>()); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().setSerializationLib(MetadataTypedColumnsetSerDe.class.getName()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setInputFormat(SequenceFileInputFormat.class.getName()); sd.setOutputFormat(HiveSequenceFileOutputFormat.class.getName()); org.apache.hadoop.hive.metastore.api.Table t = new org.apache.hadoop.hive.metastore.api.Table(); t.setSd(sd); t.setPartitionKeys(new ArrayList<FieldSchema>()); t.setParameters(new HashMap<String, String>()); t.setTableType(TableType.MANAGED_TABLE.toString()); t.setDbName(databaseName); t.setTableName(tableName);
fields.add(new FieldSchema("colname", serdeConstants.STRING_TYPE_NAME, "")); Table tbl = new Table(); tbl.setDbName(dbName); tbl.setTableName(tblName); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(Lists.newArrayList(new FieldSchema("data_column", serdeConstants.STRING_TYPE_NAME, ""))); tbl.setSd(sd); sd.setInputFormat(RCFileInputFormat.class.getName()); sd.setOutputFormat(RCFileOutputFormat.class.getName()); sd.setParameters(new HashMap<String, String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib( org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); tbl.setPartitionKeys(fields);
@Test public void testCreateTableDefaultValuesView() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(DEFAULT_DATABASE); table.setTableName("test_table_2"); table.setTableType("VIRTUAL_VIEW"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); // No location should be created for views Assert.assertNull("Storage descriptor location should be null", createdTable.getSd().getLocation()); }
viewCols.add(new FieldSchema("income", ColumnType.INT_TYPE_NAME, "")); viewPartitionCols.add(new FieldSchema("name", ColumnType.STRING_TYPE_NAME, "")); Table view = new Table(); view.setDbName(dbName); view.setTableName(viewName); view.setTableType(TableType.VIRTUAL_VIEW.name()); view.setPartitionKeys(viewPartitionCols); view.setViewOriginalText("SELECT income, name FROM " + tblName); "`.`name` FROM `" + dbName + "`.`" + tblName + "`"); view.setRewriteEnabled(false); StorageDescriptor viewSd = new StorageDescriptor(); view.setSd(viewSd); viewSd.setCols(viewCols); viewSd.setCompressed(false); viewSd.setParameters(new HashMap<>()); viewSd.setSerdeInfo(new SerDeInfo()); viewSd.getSerdeInfo().setParameters(new HashMap<>()); vals.add("abc"); Partition part = new Partition(); part.setDbName(dbName); part.setTableName(viewName); part.setValues(vals); part.setParameters(new HashMap<>());
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbname, ImportTableDesc tblDesc, Partition partition) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); return partsDesc; }
Partition part = new Partition(); orderedResult.add(part); part.setParameters(new HashMap<>()); part.setValues(new ArrayList<String>()); part.setCatName(catName); part.setDbName(dbName); StorageDescriptor sd = new StorageDescriptor(); StorageDescriptor oldSd = sds.put(sdId, sd); if (oldSd != null) { sd.setSortCols(new ArrayList<Order>()); sd.setBucketCols(new ArrayList<String>()); sd.setParameters(new HashMap<String, String>()); sd.setSkewedInfo(new SkewedInfo(new ArrayList<String>(), SerDeInfo serde = new SerDeInfo(); SerDeInfo oldSerde = serdes.put(serdeId, serde); if (oldSerde != null) { throw new MetaException("SDs reuse serdes; we don't expect that"); serde.setParameters(new HashMap<String, String>()); serde.setName((String)fields[12]); serde.setSerializationLib((String)fields[13]); serdeSb.append(serdeId).append(","); sd.setSerdeInfo(serde);
needsLocation = doesTableNeedLocation(tbl); if (tbl.isView()) { String createTab_stmt = "CREATE VIEW `" + tableName + "` AS " + tbl.getViewExpandedText(); outStream.write(createTab_stmt.getBytes(StandardCharsets.UTF_8)); return 0; if (tbl.isTemporary()) { duplicateProps.add("TEMPORARY"); tbl_temp = "TEMPORARY "; StorageDescriptor sd = tbl.getTTable().getSd(); SerDeInfo serdeInfo = sd.getSerdeInfo(); Map<String, String> serdeParams = serdeInfo.getParameters(); tbl_row_format.append("ROW FORMAT SERDE \n"); tbl_row_format.append(" '" + HiveStringUtils.escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n"); if (tbl.getStorageHandler() == null) { + HiveStringUtils.escapeHiveCommand(sd.getInputFormat()) + "' \n"); tbl_row_format.append("OUTPUTFORMAT \n '" + HiveStringUtils.escapeHiveCommand(sd.getOutputFormat()) + "'"); } else { duplicateProps.add(META_TABLE_STORAGE); appendSerdeParams(tbl_row_format, serdeInfo.getParameters());
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception { StorageDescriptor partitionSd = new StorageDescriptor(); if (StringUtils.isNotBlank(tbl.getSd().getLocation())) { partitionSd.setLocation(tbl.getSd().getLocation() + values); } else { partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1"); } partitionSd.setSerdeInfo( new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"))); partitionSd.setCols(tbl.getPartitionKeys()); Partition partition = new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>()); partition.setCreateTime(createTime); return this.getLocalMetastoreClient().add_partition(partition); }
private void addSd(ArrayList<FieldSchema> cols, Table tbl) { StorageDescriptor sd = new StorageDescriptor(); sd.setCols(cols); sd.setCompressed(false); sd.setNumBuckets(1); sd.setParameters(new HashMap<String, String>()); sd.setBucketCols(new ArrayList<String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters() .put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); }
private Table createPartitionedTable(String catName, String dbName, String tableName) throws Exception { try { db.dropTable(catName, dbName, tableName); Table table = new Table(); table.setCatName(catName); table.setDbName(dbName); table.setTableName(tableName); FieldSchema col1 = new FieldSchema("key", "string", ""); FieldSchema col2 = new FieldSchema("value", "int", ""); FieldSchema col3 = new FieldSchema("city", "string", ""); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo()); sd.setInputFormat(TextInputFormat.class.getCanonicalName()); sd.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); sd.setCols(Arrays.asList(col1, col2)); table.setPartitionKeys(Arrays.asList(col3)); table.setSd(sd); db.createTable(table); return db.getTable(catName, dbName, tableName); } catch (Exception exception) { fail("Unable to drop and create table " + StatsUtils.getFullyQualifiedTableName(dbName, tableName) + " because " + StringUtils.stringifyException(exception)); throw exception; } }
@Test public void testGetTableAvro() { final String databaseName = "testdb"; final String tableName = "testtable"; HiveTable.Builder builder = new HiveTable.Builder(); builder.withDbName(databaseName).withTableName(tableName); State serdeProps = new State(); serdeProps.setProp("avro.schema.literal", "{\"type\": \"record\", \"name\": \"TestEvent\"," + " \"namespace\": \"test.namespace\", \"fields\": [{\"name\":\"a\"," + " \"type\": \"int\"}]}"); builder.withSerdeProps(serdeProps); HiveTable hiveTable = builder.build(); hiveTable.setInputFormat(AvroContainerInputFormat.class.getName()); hiveTable.setOutputFormat(AvroContainerOutputFormat.class.getName()); hiveTable.setSerDeType(AvroSerDe.class.getName()); Table table = HiveMetaStoreUtils.getTable(hiveTable); Assert.assertEquals(table.getDbName(), databaseName); Assert.assertEquals(table.getTableName(), tableName); StorageDescriptor sd = table.getSd(); Assert.assertEquals(sd.getInputFormat(), AvroContainerInputFormat.class.getName()); Assert.assertEquals(sd.getOutputFormat(), AvroContainerOutputFormat.class.getName()); Assert.assertNotNull(sd.getSerdeInfo()); Assert.assertEquals(sd.getSerdeInfo().getSerializationLib(), AvroSerDe.class.getName()); List<FieldSchema> fields = sd.getCols(); Assert.assertTrue(fields != null && fields.size() == 1); FieldSchema fieldA = fields.get(0); Assert.assertEquals(fieldA.getName(), "a"); Assert.assertEquals(fieldA.getType(), "int"); }