public List<String> getTimePartColNamesOfTable(String storageTableName) throws LensException { return getTimePartColNamesOfTable(getTable(storageTableName)); }
public List<String> getTimePartColNamesOfTable(String storageTableName) throws LensException { return getTimePartColNamesOfTable(getTable(storageTableName)); }
private void loadTimelinesFromTableProperties(UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws HiveException, LensException { log.info("loading from table properties: {}", storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTableName)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol).init(getTable(storageTableName)); } }
private void loadTimelinesFromTableProperties(UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws HiveException, LensException { log.info("loading from table properties: {}", storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTableName)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol).init(getTable(storageTableName)); } }
private void ensureEntryForTimeLineKey(String fact, String storage, UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws LensException { // Not found in table properties either, compute from all partitions of the fact-storage table. // First make sure all combinations of update period and partition column have an entry even // if no partitions exist if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) { log.info("loading from all partitions: {}", storageTableName); Table storageTable = getTable(storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTable)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol); } } }
private void ensureEntryForTimeLineKey(String fact, String storage, UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws LensException { // Not found in table properties either, compute from all partitions of the fact-storage table. // First make sure all combinations of update period and partition column have an entry even // if no partitions exist if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) { log.info("loading from all partitions: {}", storageTableName); Table storageTable = getTable(storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTable)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol); } } }
@Override public XPartitionList getAllPartitionsOfFactStorage(LensSessionHandle sessionid, String fact, String storageName, String filter) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { checkFactStorage(sessionid, fact, storageName); CubeMetastoreClient client = getClient(sessionid); Set<String> storageTableNames = getAllTablesForStorage(sessionid, fact, storageName); List<Partition> parts = new ArrayList<>(); List<String> timePartCols = new ArrayList<>(); for (String storageTableName : storageTableNames) { parts.addAll(client.getPartitionsByFilter(storageTableName, filter)); timePartCols.addAll(client.getTimePartColNamesOfTable(storageTableName)); } return xpartitionListFromPartitionList(fact, parts, timePartCols); } catch (HiveException exc) { throw new LensException(exc); } }
private void loadTimelinesFromAllPartitions(String storageTableName, String timeLineKey) throws HiveException, LensException { // Then add all existing partitions for batch addition in respective timelines. Table storageTable = getTable(storageTableName); List<String> timeParts = getTimePartColNamesOfTable(storageTable); List<FieldSchema> partCols = storageTable.getPartCols(); for (Partition partition : getPartitionsByFilter(storageTableName, null)) { UpdatePeriod period = deduceUpdatePeriod(partition); List<String> values = partition.getValues(); if (values.contains(StorageConstants.LATEST_PARTITION_VALUE)) { log.info("dropping latest partition from fact storage table: {}. Spec: {}", storageTableName, partition.getSpec()); getClient().dropPartition(storageTableName, values, false); continue; } for (int i = 0; i < partCols.size(); i++) { if (timeParts.contains(partCols.get(i).getName())) { addForBatchAddition(timeLineKey, storageTableName, period, partCols.get(i).getName(), values.get(i)); } } } }
private void loadTimelinesFromAllPartitions(String storageTableName, String timeLineKey) throws HiveException, LensException { // Then add all existing partitions for batch addition in respective timelines. Table storageTable = getTable(storageTableName); List<String> timeParts = getTimePartColNamesOfTable(storageTable); List<FieldSchema> partCols = storageTable.getPartCols(); for (Partition partition : getPartitionsByFilter(storageTableName, null)) { UpdatePeriod period = deduceUpdatePeriod(partition); List<String> values = partition.getValues(); if (values.contains(StorageConstants.LATEST_PARTITION_VALUE)) { log.info("dropping latest partition from fact storage table: {}. Spec: {}", storageTableName, partition.getSpec()); getClient().dropPartition(storageTableName, values, false); continue; } for (int i = 0; i < partCols.size(); i++) { if (timeParts.contains(partCols.get(i).getName())) { addForBatchAddition(timeLineKey, storageTableName, period, partCols.get(i).getName(), values.get(i)); } } } }
List<String> timePartCols = getTimePartColNamesOfTable(storageTable); List<Partition> latestParts = Lists.newArrayList(); for (Partition partition : partitions) {
List<String> timePartCols = getTimePartColNamesOfTable(storageTable); List<Partition> latestParts = Lists.newArrayList(); for (Partition partition : partitions) {
@Override public XPartitionList getAllPartitionsOfDimTableStorage( LensSessionHandle sessionid, String dimTable, String storageName, String filter) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ checkDimTableStorage(sessionid, dimTable, storageName); CubeMetastoreClient client = getClient(sessionid); String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTable, storageName); List<Partition> partitions = client.getPartitionsByFilter(storageTableName, filter); List<String> timePartCols = client.getTimePartColNamesOfTable(storageTableName); return xpartitionListFromPartitionList(dimTable, partitions, timePartCols); } catch (HiveException exc) { throw new LensException(exc); } }