public Path getDataLocation() { if (table.isPartitioned()) { if (tPartition.getSd() == null) return null; else return new Path(tPartition.getSd().getLocation()); } else { if (table.getTTable() == null || table.getTTable().getSd() == null) return null; else return new Path(table.getTTable().getSd().getLocation()); } }
protected Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); desc.getTTable().getSd().setLocation(dummyPath.toString()); desc.getTTable().getSd().getSerdeInfo().setSerializationLib(NullStructSerDe.class.getName()); desc.setInputFormatClass(NullRowsInputFormat.class); desc.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); return desc; }
private SerDeInfo getSerDeInfo() { if (!sd.isSetSerdeInfo()) { sd.setSerdeInfo(new SerDeInfo()); } return sd.getSerdeInfo(); }
StorageDescriptorKey(String baseLocation, StorageDescriptor sd) { this.sd = sd; this.baseLocation = baseLocation; if (sd == null) { hashCode = Objects.hashCode(baseLocation); } else { // use the baseLocation provided instead of sd.getLocation() hashCode = Objects.hash(sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getSerializationLib(), sd.getInputFormat(), sd.getOutputFormat(), baseLocation, sd.getCols()); } }
public HCatTable(String dbName, String tableName) { this.dbName = StringUtils.isBlank(dbName)? Warehouse.DEFAULT_DATABASE_NAME : dbName; this.tableName = tableName; this.sd = new StorageDescriptor(); this.sd.setInputFormat(DEFAULT_INPUT_FORMAT_CLASS); this.sd.setOutputFormat(DEFAULT_OUTPUT_FORMAT_CLASS); this.sd.setSerdeInfo(new SerDeInfo()); this.sd.getSerdeInfo().setSerializationLib(DEFAULT_SERDE_CLASS); this.sd.getSerdeInfo().setParameters(new HashMap<String, String>()); this.sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); // Default serialization format. }
@Test public void testAddPartitionSpecSetRootPath() throws Exception { Table table = createTable(); String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; String rootPath1 = table.getSd().getLocation() + "/someotherpath/"; Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath + "part2007/"); PartitionSpecProxy partitionSpecProxy = buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath1, Lists.newArrayList(partition)); client.add_partitions_pspec(partitionSpecProxy); Partition resultPart = client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList("2007")); Assert.assertEquals(rootPath + "part2007", resultPart.getSd().getLocation()); }
@Test public void testDropTableExternalWithoutPurge() throws Exception { Table table = externalTable; client.dropTable(table.getDbName(), table.getTableName(), true, true, false); Assert.assertTrue("Table path should not be removed", metaStore.isPathExists(new Path(table.getSd().getLocation()))); Assert.assertFalse("Table path should be in trash", metaStore.isPathExistsInTrash(new Path(table.getSd().getLocation()))); }
@Test public void testDropPartitionDeleteParentDir() throws Exception { client.dropPartition(DB_NAME, TABLE_NAME, PARTITIONS[0].getValues(), true); client.dropPartition(DB_NAME, TABLE_NAME, PARTITIONS[1].getValues(), true); List<Partition> droppedPartitions = Lists.newArrayList(PARTITIONS[0], PARTITIONS[1]); List<Partition> remainingPartitions = Lists.newArrayList(PARTITIONS[2]); checkPartitionsAfterDelete(TABLE_NAME, droppedPartitions, remainingPartitions, true, false); Path parentPath = new Path(PARTITIONS[0].getSd().getLocation()).getParent(); Assert.assertFalse("The parent path '" + parentPath.toString() + "' should not exist.", metaStore.isPathExists(parentPath)); }
private String getFinalDynamicPartitionDestination(Table table, Map<String, String> partKVs, OutputJobInfo jobInfo) { Path partPath = new Path(table.getTTable().getSd().getLocation()); if (!customDynamicLocationUsed) { // file:///tmp/hcat_junit_warehouse/employee/_DYN0.7770480401313761/emp_country=IN/emp_state=KA -> // file:///tmp/hcat_junit_warehouse/employee/emp_country=IN/emp_state=KA for (FieldSchema partKey : table.getPartitionKeys()) { partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs); } return partPath.toString(); } else { // if custom root specified, update the parent path if (jobInfo.getCustomDynamicRoot() != null && jobInfo.getCustomDynamicRoot().length() > 0) { partPath = new Path(partPath, jobInfo.getCustomDynamicRoot()); } return new Path(partPath, HCatFileUtil.resolveCustomPath(jobInfo, partKVs, false)).toString(); } }
@Test public void testAddPartitionNullLocationInTableToo() throws Exception { createTable(DB_NAME, TABLE_NAME, null); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, null); client.add_partition(partition); Partition part = client.getPartition(DB_NAME, TABLE_NAME, "year=2017"); Assert.assertEquals( metaStore.getWarehouseRoot() + "/test_partition_db.db/test_partition_table/year=2017", part.getSd().getLocation()); Assert.assertTrue(metaStore.isPathExists(new Path(part.getSd().getLocation()))); }
private Path getPartitionLocation() throws NoSuchObjectException, MetaException, TException { Path partitionLocacation; if (partition.isEmpty()) { partitionLocacation = new Path(table.getSd().getLocation()); } else { // TODO: calculate this instead. Just because we're writing to the location doesn't mean that it'll // always be wanted in the meta store right away. List<Partition> partitionEntries = metaStoreClient.listPartitions(table.getDbName(), table.getTableName(), partition, (short) 1); partitionLocacation = new Path(partitionEntries.get(0).getSd().getLocation()); } return partitionLocacation; }
private void verifyPartitionSharedSD(Table table, String name, List<String> values, int index) throws Exception { Partition part = client.getPartition(table.getDbName(), table.getTableName(), name); Assert.assertNotNull(part); Assert.assertEquals(table.getTableName(), part.getTableName()); List<String> partValues = part.getValues(); Assert.assertEquals(values.size(), partValues.size()); Assert.assertTrue(partValues.containsAll(values)); Assert.assertEquals(table.getDbName(), part.getDbName()); Assert.assertEquals(DEFAULT_CREATE_TIME, part.getLastAccessTime()); Assert.assertEquals(DEFAULT_PARAM_VALUE + index, part.getParameters().get(DEFAULT_PARAM_KEY + index)); Assert.assertFalse(part.getParameters().keySet().contains(table.getParameters().keySet())); StorageDescriptor sd = part.getSd(); Assert.assertNotNull(sd); Assert.assertEquals("TestInputFormat", sd.getInputFormat()); Assert.assertEquals("TestOutputFormat", sd.getOutputFormat()); Assert.assertEquals("sharedSDPartSerde", sd.getSerdeInfo().getName()); Assert.assertEquals("testSDParamValue", sd.getParameters().get("testSDParamKey")); Assert.assertEquals( metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/partwithoutsd" + index, sd.getLocation()); Assert.assertTrue(metaStore.isPathExists(new Path(sd.getLocation()))); }
private PartitionHelper newWarehousePartitionHelper() throws MetaException, WorkerException { String location = table.getTable().getSd().getLocation(); Path tablePath = new Path(location); List<FieldSchema> partitionFields = table.getTable().getPartitionKeys(); List<String> partitionColumns = new ArrayList<>(partitionFields.size()); for (FieldSchema field : partitionFields) { partitionColumns.add(field.getName()); } return new WarehousePartitionHelper(configuration, tablePath, partitionColumns); }
@Test public void testTruncateTableUnpartitioned() throws Exception { // Unpartitioned table Path dataFile = new Path(testTables[0].getSd().getLocation() + "/dataFile"); client.truncateTable(testTables[0].getDbName(), testTables[0].getTableName(), null); Assert.assertTrue("Location should exist", metaStore.isPathExists(new Path(testTables[0].getSd().getLocation()))); Assert.assertFalse("DataFile should be removed", metaStore.isPathExists(dataFile)); }
private static Partition makePartitionObject(String dbName, String tblName, List<String> ptnVals, Table tbl, String ptnLocationSuffix) throws MetaException { Partition part4 = new Partition(); part4.setDbName(dbName); part4.setTableName(tblName); part4.setValues(ptnVals); part4.setParameters(new HashMap<>()); part4.setSd(tbl.getSd().deepCopy()); part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix); MetaStoreServerUtils.updatePartitionStatsFast(part4, tbl, warehouse, false, false, null, true); return part4; }
@Test public void testCreateTableDefaultValuesView() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(DEFAULT_DATABASE); table.setTableName("test_table_2"); table.setTableType("VIRTUAL_VIEW"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); // No location should be created for views Assert.assertNull("Storage descriptor location should be null", createdTable.getSd().getLocation()); }
private static void addPartition(IMetaStoreClient client, Table tbl , List<String> partValues) throws IOException, TException { Partition part = new Partition(); part.setDbName(tbl.getDbName()); part.setTableName(tbl.getTableName()); StorageDescriptor sd = new StorageDescriptor(tbl.getSd()); sd.setLocation(sd.getLocation() + Path.SEPARATOR + makePartPath(tbl.getPartitionKeys(), partValues)); part.setSd(sd); part.setValues(partValues); client.add_partition(part); }
private void alterPartitionSpecInMemory(Table tbl, Map<String, String> partSpec, org.apache.hadoop.hive.metastore.api.Partition tpart, boolean inheritTableSpecs, String partPath) throws HiveException, InvalidOperationException { LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); if (inheritTableSpecs) { tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); tpart.getSd().getSerdeInfo().setParameters( tbl.getTTable().getSd().getSerdeInfo().getParameters()); tpart.getSd().setBucketCols(tbl.getBucketCols()); tpart.getSd().setNumBuckets(tbl.getNumBuckets()); tpart.getSd().setSortCols(tbl.getSortCols()); } if (partPath == null || partPath.trim().equals("")) { throw new HiveException("new partition path should not be null or empty."); } tpart.getSd().setLocation(partPath); }
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception { StorageDescriptor partitionSd = new StorageDescriptor(); if (StringUtils.isNotBlank(tbl.getSd().getLocation())) { partitionSd.setLocation(tbl.getSd().getLocation() + values); } else { partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1"); } partitionSd.setSerdeInfo( new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"))); partitionSd.setCols(tbl.getPartitionKeys()); Partition partition = new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>()); partition.setCreateTime(createTime); return this.getLocalMetastoreClient().add_partition(partition); }