/** extract update period from partition properties */ private UpdatePeriod deduceUpdatePeriod(Partition partition) { return UpdatePeriod.valueOf(partition.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD)); }
/** extract update period from partition properties */ private UpdatePeriod deduceUpdatePeriod(Partition partition) { return UpdatePeriod.valueOf(partition.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD)); }
public static void filterPartitionsByUpdatePeriod(List<Partition> partitions, UpdatePeriod updatePeriod) { Iterator<Partition> iter = partitions.iterator(); while (iter.hasNext()) { Partition part = iter.next(); if (!UpdatePeriod.valueOf(part.getParameters().get(PARTITION_UPDATE_PERIOD)).equals(updatePeriod)) { iter.remove(); } } }
public static void filterPartitionsByUpdatePeriod(List<Partition> partitions, UpdatePeriod updatePeriod) { Iterator<Partition> iter = partitions.iterator(); while (iter.hasNext()) { Partition part = iter.next(); if (!UpdatePeriod.valueOf(part.getParameters().get(PARTITION_UPDATE_PERIOD)).equals(updatePeriod)) { iter.remove(); } } }
StorageTableResolver(Configuration conf) { this.conf = conf; this.supportedStorages = getSupportedStorages(conf); this.allStoragesSupported = (supportedStorages == null); this.failOnPartialData = conf.getBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); String str = conf.get(CubeQueryConfUtil.VALID_STORAGE_DIM_TABLES); validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); String maxIntervalStr = conf.get(CubeQueryConfUtil.QUERY_MAX_INTERVAL); if (maxIntervalStr != null) { this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr.toUpperCase()); } else { this.maxInterval = null; } this.phase = PHASE.first(); }
private static Map<String, UpdatePeriod> getDumpPeriods(String name, Map<String, String> params) { String storagesStr = params.get(MetastoreUtil.getDimensionStorageListKey(name)); if (!StringUtils.isBlank(storagesStr)) { Map<String, UpdatePeriod> dumpPeriods = new HashMap<>(); for (String storage : StringUtils.split(storagesStr, ",")) { String dumpPeriod = params.get(MetastoreUtil.getDimensionDumpPeriodKey(name, storage)); if (dumpPeriod != null) { dumpPeriods.put(storage, UpdatePeriod.valueOf(dumpPeriod)); } else { dumpPeriods.put(storage, null); } } return dumpPeriods; } return null; }
StorageTableResolver(Configuration conf) { this.conf = conf; this.supportedStorages = getSupportedStorages(conf); this.allStoragesSupported = (supportedStorages == null); this.failOnPartialData = conf.getBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); String str = conf.get(CubeQueryConfUtil.VALID_STORAGE_DIM_TABLES); validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); String maxIntervalStr = conf.get(CubeQueryConfUtil.QUERY_MAX_INTERVAL); if (maxIntervalStr != null) { this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr.toUpperCase()); } else { this.maxInterval = null; } this.phase = PHASE.first(); }
private static Map<String, UpdatePeriod> getDumpPeriods(String name, Map<String, String> params) { String storagesStr = params.get(MetastoreUtil.getDimensionStorageListKey(name)); if (!StringUtils.isBlank(storagesStr)) { Map<String, UpdatePeriod> dumpPeriods = new HashMap<>(); for (String storage : StringUtils.split(storagesStr, ",")) { String dumpPeriod = params.get(MetastoreUtil.getDimensionDumpPeriodKey(name, storage)); if (dumpPeriod != null) { dumpPeriods.put(storage, UpdatePeriod.valueOf(dumpPeriod)); } else { dumpPeriods.put(storage, null); } } return dumpPeriods; } return null; }
private Date getPartDate(Partition part, int timeColIndex) { String partVal = part.getValues().get(timeColIndex); String updatePeriodStr = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); Date partDate = null; if (updatePeriodStr != null) { UpdatePeriod partInterval = UpdatePeriod.valueOf(updatePeriodStr); try { partDate = partInterval.parse(partVal); } catch (ParseException e) { // ignore } } return partDate; }
private Date getPartDate(Partition part, int timeColIndex) { String partVal = part.getValues().get(timeColIndex); String updatePeriodStr = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); Date partDate = null; if (updatePeriodStr != null) { UpdatePeriod partInterval = UpdatePeriod.valueOf(updatePeriodStr); try { partDate = partInterval.parse(partVal); } catch (ParseException e) { // ignore } } return partDate; }
public static Date getLatestTimeStampFromPartition(Partition part, String partCol) throws HiveException { if (part != null) { String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol)); String latestPartUpdatePeriod = part.getParameters().get(PARTITION_UPDATE_PERIOD); UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase()); try { return latestUpdatePeriod.parse(latestTimeStampStr); } catch (ParseException e) { throw new HiveException(e); } } return null; }
public static Date getLatestTimeStampFromPartition(Partition part, String partCol) throws HiveException { if (part != null) { String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol)); String latestPartUpdatePeriod = part.getParameters().get(PARTITION_UPDATE_PERIOD); UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase()); try { return latestUpdatePeriod.parse(latestTimeStampStr); } catch (ParseException e) { throw new HiveException(e); } } return null; }
public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { UpdatePeriod dumpPeriod = null; if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) { dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name()); } dumpPeriods.put(ste.getStorageName(), dumpPeriod); } return dumpPeriods; } return null; }
public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { UpdatePeriod dumpPeriod = null; if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) { dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name()); } dumpPeriods.put(ste.getStorageName(), dumpPeriod); } return dumpPeriods; } return null; }
public static Map<String, String> getFullPartSpecAsMap(XPartition partition) { Map<String, String> spec = Maps.newHashMap(); if (partition.getTimePartitionSpec() != null) { for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) { spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format() .format(getDateFromXML(timePartSpecElement.getValue()))); } } if (partition.getNonTimePartitionSpec() != null) { for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) { spec.put(partSpecElement.getKey(), partSpecElement.getValue()); } } return spec; } }
public static Map<String, String> getFullPartSpecAsMap(XPartition partition) { Map<String, String> spec = Maps.newHashMap(); if (partition.getTimePartitionSpec() != null) { for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) { spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format() .format(getDateFromXML(timePartSpecElement.getValue()))); } } if (partition.getNonTimePartitionSpec() != null) { for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) { spec.put(partSpecElement.getKey(), partSpecElement.getValue()); } } return spec; } }
@Override public void addDimTableStorage(LensSessionHandle sessionid, String dimTblName, XStorageTableElement storageTable) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); UpdatePeriod period = null; if (storageTable.getUpdatePeriods() != null && !storageTable.getUpdatePeriods().getUpdatePeriod().isEmpty()) { period = UpdatePeriod.valueOf(storageTable.getUpdatePeriods().getUpdatePeriod().get(0).name()); } msClient.addStorage(dimTable, storageTable.getStorageName(), period, JAXBUtils.storageTableDescFromXStorageTableDesc(storageTable.getTableDesc())); log.info("Added storage " + storageTable.getStorageName() + " for dimension table " + dimTblName + " with update period " + period); } }
@Override public void updatePartition(LensSessionHandle sessionid, String tblName, String storageName, XPartition xPartition) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient client = getClient(sessionid); String storageTableName = client .getStorageTableName(tblName, storageName, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().name())); Partition existingPartition = client.getPartitionByFilter(storageTableName, StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition))); JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition); client.updatePartition(tblName, storageName, existingPartition, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().value())); } catch (HiveException | ClassNotFoundException | InvalidOperationException | UnsupportedOperationException exc) { throw new LensException(exc); } }
public static StoragePartitionDesc storagePartSpecFromXPartition( XPartition xpart) { StoragePartitionDesc partDesc = new StoragePartitionDesc( xpart.getFactOrDimensionTableName(), timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()), nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()), UpdatePeriod.valueOf(xpart.getUpdatePeriod().name())); partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters())); partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters())); partDesc.setLocation(xpart.getLocation()); partDesc.setInputFormat(xpart.getInputFormat()); partDesc.setOutputFormat(xpart.getOutputFormat()); partDesc.setSerializationLib(xpart.getSerdeClassname()); return partDesc; }
public static StoragePartitionDesc storagePartSpecFromXPartition( XPartition xpart) { StoragePartitionDesc partDesc = new StoragePartitionDesc( xpart.getFactOrDimensionTableName(), timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()), nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()), UpdatePeriod.valueOf(xpart.getUpdatePeriod().name())); partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters())); partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters())); partDesc.setLocation(xpart.getLocation()); partDesc.setInputFormat(xpart.getInputFormat()); partDesc.setOutputFormat(xpart.getOutputFormat()); partDesc.setSerializationLib(xpart.getSerdeClassname()); return partDesc; }