private void dropPartitionsIteratively(String dbName, String tableName, Map<String, String> partitionSpec, boolean ifExists, boolean deleteData) throws HCatException, TException { LOG.info("HCatClient: Dropping partitions iteratively."); List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName, getFilterString(partitionSpec), (short) -1); for (Partition partition : partitions) { dropPartition(partition, ifExists, deleteData); } }
@Test(expected = NoSuchObjectException.class) public void testListPartitionsByFilterNoTable() throws Exception { client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"", (short)-1); }
/** * Get a list of {@link Partition}s for the <code>table</code> that matches an optional <code>filter</code> * * @param client an {@link IMetaStoreClient} for the correct metastore. * @param table the {@link Table} for which we should get partitions. * @param filter an optional filter for partitions as would be used in Hive. Can only filter on String columns. * (e.g. "part = \"part1\"" or "date > \"2015\"". * @return a list of {@link Partition}s */ public static List<Partition> getPartitions(IMetaStoreClient client, Table table, Optional<String> filter, Optional<? extends HivePartitionExtendedFilter> hivePartitionExtendedFilterOptional) throws IOException { try { List<Partition> partitions = Lists.newArrayList(); List<org.apache.hadoop.hive.metastore.api.Partition> partitionsList = filter.isPresent() ? client.listPartitionsByFilter(table.getDbName(), table.getTableName(), filter.get(), (short) -1) : client.listPartitions(table.getDbName(), table.getTableName(), (short) -1); for (org.apache.hadoop.hive.metastore.api.Partition p : partitionsList) { if (!hivePartitionExtendedFilterOptional.isPresent() || hivePartitionExtendedFilterOptional.get().accept(p)) { Partition partition = new Partition(table, p); partitions.add(partition); } } return partitions; } catch (TException | HiveException te) { throw new IOException("Hive Error", te); } }
@Test(expected = NoSuchObjectException.class) public void listPartitionsByFilterBogusCatalog() throws TException { createTable3PartCols1Part(client); client.listPartitionsByFilter("bogus", DB_NAME, TABLE_NAME, "partcol=\"a0\"", -1); }
/** * Testing listPartitionsByFilter(String,String,String,short) -> * get_partitions_by_filter(String,String,String,short). */ @Test public void testListPartitionsByFilter() throws Exception { List<List<String>> partValues = createTable4PartColsParts(client); List<Partition> partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\" OR " + "mm=\"02\"", (short)-1); assertEquals(3, partitions.size()); assertPartitionsHaveCorrectValues(partitions, partValues.subList(1, 4)); partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\" OR " + "mm=\"02\"", (short)2); assertEquals(2, partitions.size()); assertPartitionsHaveCorrectValues(partitions, partValues.subList(1, 3)); partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\" OR " + "mm=\"02\"", (short)0); assertTrue(partitions.isEmpty()); partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\" AND mm=\"99\"", (short)-1); assertTrue(partitions.isEmpty()); }
@Test(expected = NoSuchObjectException.class) public void testListPartitionsByFilterNoDbName() throws Exception { createTable4PartColsParts(client); client.listPartitionsByFilter("", TABLE_NAME, "yyyy=\"2017\"", (short)-1); }
@Test(expected = MetaException.class) public void testListPartitionsByFilterHighMaxParts() throws Exception { createTable4PartColsParts(client); client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"", (short)101); }
@Test(expected = NoSuchObjectException.class) public void testListPartitionsByFilterNoDb() throws Exception { client.dropDatabase(DB_NAME); client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"", (short)-1); }
@Test(expected = MetaException.class) public void testListPartitionsByFilterInvalidFilter() throws Exception { createTable4PartColsParts(client); client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyy=\"2017\"", (short)101); }
@Test(expected = NoSuchObjectException.class) public void testListPartitionsByFilterNoTblName() throws Exception { createTable4PartColsParts(client); client.listPartitionsByFilter(DB_NAME, "", "yyyy=\"2017\"", (short)-1); }
@Test public void testListPartitionsByFilterNullFilter() throws Exception { createTable4PartColsParts(client); List<Partition> partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, null, (short)-1); assertEquals(4, partitions.size()); }
@Override public List<HCatPartition> listPartitionsByFilter(String dbName, String tblName, String filter) throws HCatException { List<HCatPartition> hcatPtns = new ArrayList<HCatPartition>(); try { HCatTable table = getTable(dbName, tblName); List<Partition> hivePtns = hmsClient.listPartitionsByFilter( table.getDbName(), table.getTableName(), filter, (short) -1); for (Partition ptn : hivePtns) { hcatPtns.add(new HCatPartition(table, ptn)); } } catch (MetaException e) { throw new HCatException("MetaException while fetching partitions.", e); } catch (NoSuchObjectException e) { throw new ObjectNotFoundException( "NoSuchObjectException while fetching partitions.", e); } catch (TException e) { throw new ConnectionFailureException( "TException while fetching partitions.", e); } return hcatPtns; }
@Test public void testListPartitionsByFilterEmptyFilter() throws Exception { createTable4PartColsParts(client); List<Partition> partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "", (short)-1); assertEquals(4, partitions.size()); }
@Test public void testListPartitionsByFilterCaseSensitive() throws Exception { String tableName = TABLE_NAME + "_casesensitive"; Table t = createTestTable(client, DB_NAME, tableName, Lists.newArrayList("yyyy", "month", "day"), false); List<List<String>> testValues = Lists.newArrayList( Lists.newArrayList("2017", "march", "11"), Lists.newArrayList("2017", "march", "15"), Lists.newArrayList("2017", "may", "15"), Lists.newArrayList("2018", "march", "11"), Lists.newArrayList("2018", "april", "7")); for(List<String> vals : testValues) { addPartition(client, t, vals); } List<Partition> partitions = client.listPartitionsByFilter(DB_NAME, tableName, "month=\"mArCh\"", (short) -1); Assert.assertTrue(partitions.isEmpty()); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "yyyy=\"2017\" AND month=\"May\"", (short) -1); Assert.assertTrue(partitions.isEmpty()); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "yyyy=\"2017\" AND month!=\"mArCh\"", (short) -1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 3)); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "month like \"M.*\"", (short) -1); Assert.assertTrue(partitions.isEmpty()); client.dropTable(DB_NAME, tableName); }
List<Partition> partitions = client.listPartitionsByFilter(DB_NAME, tableName, "yYyY=\"2017\"", (short) -1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 3)); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "yYyY=\"2017\" AND mOnTh=\"may\"", (short) -1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(2, 3)); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "yYyY!=\"2017\"", (short) -1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(3, 5)); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "mOnTh=\"september\"", (short) -1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(4, 5)); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "mOnTh like \"m.*\"", (short) -1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 4)); partitions = client.listPartitionsByFilter(DB_NAME, tableName, "yYyY=\"2018\" AND mOnTh like \"m.*\"", (short) -1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(3, 4));
@Test public void testListPartitionsByFilterNullTblName() throws Exception { try { createTable4PartColsParts(client); client.listPartitionsByFilter(DB_NAME, null, "yyyy=\"2017\"", (short)-1); fail("Should have thrown exception"); } catch (NullPointerException | TTransportException e) { //TODO: should not throw different exceptions for different HMS deployment types } }
@Test public void testListPartitionsByFilterNullDbName() throws Exception { try { createTable4PartColsParts(client); client.listPartitionsByFilter(null, TABLE_NAME, "yyyy=\"2017\"", (short)-1); fail("Should have thrown exception"); } catch (NullPointerException | TTransportException e) { //TODO: should not throw different exceptions for different HMS deployment types } }
/** * Get a list of Partitions by filter. * @param tbl The table containing the partitions. * @param filter A string represent partition predicates. * @return a list of partitions satisfying the partition predicates. * @throws HiveException * @throws MetaException * @throws NoSuchObjectException * @throws TException */ public List<Partition> getPartitionsByFilter(Table tbl, String filter) throws HiveException, MetaException, NoSuchObjectException, TException { if (!tbl.isPartitioned()) { throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tbl.getTableName()); } List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().listPartitionsByFilter( tbl.getDbName(), tbl.getTableName(), filter, (short)-1); return convertFromMetastore(tbl, tParts); }
/** * Get a list of Partitions by filter. * @param tbl The table containing the partitions. * @param filter A string represent partition predicates. * @return a list of partitions satisfying the partition predicates. * @throws HiveException * @throws MetaException * @throws NoSuchObjectException * @throws TException */ public List<Partition> getPartitionsByFilter(Table tbl, String filter) throws HiveException, MetaException, NoSuchObjectException, TException { if (!tbl.isPartitioned()) { throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tbl.getTableName()); } List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().listPartitionsByFilter( tbl.getDbName(), tbl.getTableName(), filter, (short)-1); return convertFromMetastore(tbl, tParts); }
Assert.assertEquals(expectedCatalog(), proxy.getCatName()); fetched = client.listPartitionsByFilter(dbName, tableName, "partcol=\"a0\"", (short)-1); Assert.assertEquals(1, fetched.size()); Assert.assertEquals(expectedCatalog(), fetched.get(0).getCatName());