public Partition getPartitionByFilter(String storageTableName, String filter) throws HiveException { List<Partition> parts = getPartitionsByFilter(storageTableName, filter); if (parts.size() != 1) { throw new HiveException( "filter " + filter + " did not result in unique partition. Got " + parts.size() + "partitions"); } return parts.iterator().next(); }
public Partition getPartitionByFilter(String storageTableName, String filter) throws HiveException { List<Partition> parts = getPartitionsByFilter(storageTableName, filter); if (parts.size() != 1) { throw new HiveException( "filter " + filter + " did not result in unique partition. Got " + parts.size() + "partitions"); } return parts.iterator().next(); }
Partition getLatestPart(String storageTableName, String latestPartCol, Map<String, String> nonTimeParts) throws HiveException { List<Partition> latestParts = getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol, nonTimeParts)); if (latestParts != null && !latestParts.isEmpty()) { return latestParts.get(0); } return null; }
Partition getLatestPart(String storageTableName, String latestPartCol, Map<String, String> nonTimeParts) throws HiveException { List<Partition> latestParts = getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol, nonTimeParts)); if (latestParts != null && !latestParts.isEmpty()) { return latestParts.get(0); } return null; }
private void assertNoPartitionNamedLatest(String storageTableName, String... partCols) throws HiveException { for (String p : partCols) { assertEquals(client.getPartitionsByFilter(storageTableName, p + "='latest'").size(), 0); } }
for (String tableName : storageTables) { String filter = getFilter(msClient, tableName, values); partitions.put(filter, msClient.getPartitionsByFilter(tableName, filter)); if (partitions.get(filter).size() > 1) { log.error("More than one partition with specified values, corresponding filter:" + filter);
private void loadTimelinesFromAllPartitions(String storageTableName, String timeLineKey) throws HiveException, LensException { // Then add all existing partitions for batch addition in respective timelines. Table storageTable = getTable(storageTableName); List<String> timeParts = getTimePartColNamesOfTable(storageTable); List<FieldSchema> partCols = storageTable.getPartCols(); for (Partition partition : getPartitionsByFilter(storageTableName, null)) { UpdatePeriod period = deduceUpdatePeriod(partition); List<String> values = partition.getValues(); if (values.contains(StorageConstants.LATEST_PARTITION_VALUE)) { log.info("dropping latest partition from fact storage table: {}. Spec: {}", storageTableName, partition.getSpec()); getClient().dropPartition(storageTableName, values, false); continue; } for (int i = 0; i < partCols.size(); i++) { if (timeParts.contains(partCols.get(i).getName())) { addForBatchAddition(timeLineKey, storageTableName, period, partCols.get(i).getName(), values.get(i)); } } } }
@Override public XPartitionList getAllPartitionsOfFactStorage(LensSessionHandle sessionid, String fact, String storageName, String filter) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { checkFactStorage(sessionid, fact, storageName); CubeMetastoreClient client = getClient(sessionid); Set<String> storageTableNames = getAllTablesForStorage(sessionid, fact, storageName); List<Partition> parts = new ArrayList<>(); List<String> timePartCols = new ArrayList<>(); for (String storageTableName : storageTableNames) { parts.addAll(client.getPartitionsByFilter(storageTableName, filter)); timePartCols.addAll(client.getTimePartColNamesOfTable(storageTableName)); } return xpartitionListFromPartitionList(fact, parts, timePartCols); } catch (HiveException exc) { throw new LensException(exc); } }
private void loadTimelinesFromAllPartitions(String storageTableName, String timeLineKey) throws HiveException, LensException { // Then add all existing partitions for batch addition in respective timelines. Table storageTable = getTable(storageTableName); List<String> timeParts = getTimePartColNamesOfTable(storageTable); List<FieldSchema> partCols = storageTable.getPartCols(); for (Partition partition : getPartitionsByFilter(storageTableName, null)) { UpdatePeriod period = deduceUpdatePeriod(partition); List<String> values = partition.getValues(); if (values.contains(StorageConstants.LATEST_PARTITION_VALUE)) { log.info("dropping latest partition from fact storage table: {}. Spec: {}", storageTableName, partition.getSpec()); getClient().dropPartition(storageTableName, values, false); continue; } for (int i = 0; i < partCols.size(); i++) { if (timeParts.contains(partCols.get(i).getName())) { addForBatchAddition(timeLineKey, storageTableName, period, partCols.get(i).getName(), values.get(i)); } } } }
CubeMetastoreClient msClient = getClient(sessionid); for (String tableName : storageTables) { partitions.addAll(msClient.getPartitionsByFilter(tableName, filter));
private void assertLatestForRegions(String storageTableName, Map<String, Date> expectedLatestValues) throws HiveException, LensException { List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), expectedLatestValues.size()); for (Partition part : parts) { assertEquals(MetastoreUtil.getLatestTimeStampFromPartition(part, getDatePartitionKey()), TimePartition.of(HOURLY, expectedLatestValues.get(part.getSpec().get("region"))).getDate()); } }
@Override public XPartitionList getAllPartitionsOfDimTableStorage( LensSessionHandle sessionid, String dimTable, String storageName, String filter) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ checkDimTableStorage(sessionid, dimTable, storageName); CubeMetastoreClient client = getClient(sessionid); String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTable, storageName); List<Partition> partitions = client.getPartitionsByFilter(storageTableName, filter); List<String> timePartCols = client.getTimePartColNamesOfTable(storageTableName); return xpartitionListFromPartitionList(dimTable, partitions, timePartCols); } catch (HiveException exc) { throw new LensException(exc); } }
TimePartition[] values = new TimePartition[partCols.length]; for (int i = 0; i < partCols.length; i++) { List<Partition> parts = client.getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(partCols[i], nonTimeParts)); for (Partition part : parts) {
String storageTableName = getFactOrDimtableStorageTableName(dimName, c1); assertEquals(client.getAllParts(storageTableName).size(), 2); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(1, parts.size()); assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName());
assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(cubeFact.getName(), c1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); assertEquals(client.getAllParts(storageTableName).size(), 1);
String storageTableName = getFactOrDimtableStorageTableName(cubeFactWithParts.getName(), c1); assertEquals(client.getAllParts(storageTableName).size(), 1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0);
assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(cubeFact.getName(), c1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); parts = client.getPartitionsByFilter(storageTableName, testDtPart.getName() + "='latest'"); assertEquals(parts.size(), 0);
String storageTableName = getFactOrDimtableStorageTableName(factName, c1); assertEquals(client.getAllParts(storageTableName).size(), 1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0);
assertEquals(client.getAllParts(storageTableName).size(), 2); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 1); Partition latestPart = parts.get(0); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 1); latestPart = parts.get(0); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 1); latestPart = parts.get(0); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timePartsNow)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 1); latestPart = parts.get(0); assertFalse(client.dimPartitionExists(cubeDim.getName(), c1, timePartsNow)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 1); latestPart = parts.get(0);