public static Date getLatestTimeStampFromPartition(Partition part, String partCol) throws HiveException { if (part != null) { String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol)); String latestPartUpdatePeriod = part.getParameters().get(PARTITION_UPDATE_PERIOD); UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase()); try { return latestUpdatePeriod.parse(latestTimeStampStr); } catch (ParseException e) { throw new HiveException(e); } } return null; }
public static Date getLatestTimeStampFromPartition(Partition part, String partCol) throws HiveException { if (part != null) { String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol)); String latestPartUpdatePeriod = part.getParameters().get(PARTITION_UPDATE_PERIOD); UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase()); try { return latestUpdatePeriod.parse(latestTimeStampStr); } catch (ParseException e) { throw new HiveException(e); } } return null; }
private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts, Map<String, TreeSet<Date>> timePartSpecs, UpdatePeriod updatePeriod) throws HiveException, LensException { Table hiveTable = getHiveTable(storageTableName); String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); if (timePartColsStr != null) { LatestInfo latest = new LatestInfo(); String[] timePartCols = StringUtils.split(timePartColsStr, ','); for (String partCol : timePartCols) { if (!timePartSpecs.containsKey(partCol)) { continue; } boolean makeLatest = true; Partition part = getLatestPart(storageTableName, partCol, nonTimeParts); Date pTimestamp = timePartSpecs.get(partCol).last(); Date latestTimestamp = getLatestTimeStampFromPartition(part, partCol); if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) { makeLatest = false; } if (makeLatest) { Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(partCol), updatePeriod.format(pTimestamp)); latest.latestParts.put(partCol, new LatestPartColumnInfo(latestParams)); } } return latest; } else { return null; } }
private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts, Map<String, TreeSet<Date>> timePartSpecs, UpdatePeriod updatePeriod) throws HiveException, LensException { Table hiveTable = getHiveTable(storageTableName); String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); if (timePartColsStr != null) { LatestInfo latest = new LatestInfo(); String[] timePartCols = StringUtils.split(timePartColsStr, ','); for (String partCol : timePartCols) { if (!timePartSpecs.containsKey(partCol)) { continue; } boolean makeLatest = true; Partition part = getLatestPart(storageTableName, partCol, nonTimeParts); Date pTimestamp = timePartSpecs.get(partCol).last(); Date latestTimestamp = getLatestTimeStampFromPartition(part, partCol); if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) { makeLatest = false; } if (makeLatest) { Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(partCol), updatePeriod.format(pTimestamp)); latest.latestParts.put(partCol, new LatestPartColumnInfo(latestParams)); } } return latest; } else { return null; } }
if (partSpec.equals(latestPartSpec)) { latestPart.getParameters().putAll(partition.getParameters()); latestPart.getParameters().put(getLatestPartTimestampKey(timePartCol), partSpec.get(timePartCol)); latestPart.getTPartition().getSd().getSerdeInfo().getParameters().putAll(
if (partSpec.equals(latestPartSpec)) { latestPart.getParameters().putAll(partition.getParameters()); latestPart.getParameters().put(getLatestPartTimestampKey(timePartCol), partSpec.get(timePartCol)); latestPart.getTPartition().getSd().getSerdeInfo().getParameters().putAll(
latest = new LatestInfo(); latest.setPart(nextLatest); Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(timeCol), nextLatest.getValues().get(timeColIndex)); latest.addLatestPartInfo(timeCol, new LatestPartColumnInfo(latestParams));
latest = new LatestInfo(); latest.setPart(nextLatest); Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(timeCol), nextLatest.getValues().get(timeColIndex)); latest.addLatestPartInfo(timeCol, new LatestPartColumnInfo(latestParams));
getLatestPartTimestampKey(partCols[i]))); if (values[i] == null || values[i].before(tp)) { values[i] = tp;
.get(addPartitionDesc.getNonTimePartSpec()).latestParts.entrySet()) { if (addPartitionDesc.getTimePartSpec().containsKey(entry.getKey()) && entry.getValue().get(MetastoreUtil.getLatestPartTimestampKey(entry.getKey())).equals( updatePeriod.format(addPartitionDesc.getTimePartSpec().get(entry.getKey())))) { if (latestPartIndexForPartCols.get(addPartitionDesc.getNonTimePartSpec()) == null) {
.get(addPartitionDesc.getNonTimePartSpec()).latestParts.entrySet()) { if (addPartitionDesc.getTimePartSpec().containsKey(entry.getKey()) && entry.getValue().get(MetastoreUtil.getLatestPartTimestampKey(entry.getKey())).equals( updatePeriod.format(addPartitionDesc.getTimePartSpec().get(entry.getKey())))) { if (latestPartIndexForPartCols.get(addPartitionDesc.getNonTimePartSpec()) == null) {
assertEquals(1, parts.size()); assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName()); assertEquals(parts.get(0).getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 0));
assertEquals(latestPart.getInputFormatClass(), TextInputFormat.class); assertFalse(latestPart.getCols().contains(newcol)); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 0)); assertEquals(latestPart.getInputFormatClass(), SequenceFileInputFormat.class); assertTrue(latestPart.getCols().contains(newcol)); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 1)); assertEquals(latestPart.getInputFormatClass(), TextInputFormat.class); assertTrue(latestPart.getCols().contains(newcol)); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 2)); latestPart = parts.get(0); assertEquals(latestPart.getInputFormatClass(), SequenceFileInputFormat.class); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 1)); assertEquals(client.getAllParts(storageTableName).size(), 3); latestPart = parts.get(0); assertEquals(latestPart.getInputFormatClass(), SequenceFileInputFormat.class); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 1)); assertEquals(client.getAllParts(storageTableName).size(), 2);