@Override public String call() { return format().format(date); } });
private Date getPartDate(Partition part, int timeColIndex) { String partVal = part.getValues().get(timeColIndex); String updatePeriodStr = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); Date partDate = null; if (updatePeriodStr != null) { UpdatePeriod partInterval = UpdatePeriod.valueOf(updatePeriodStr); try { partDate = partInterval.parse(partVal); } catch (ParseException e) { // ignore } } return partDate; }
@Override public String getName() { return name(); }
/** * Normalized cost of an update period. update period weight multiplied by number of partitions of that update period. * * @param updatePeriod * @param driver * @return normalized cost. * @throws LensException */ private double getNormalizedUpdatePeriodCost(final UpdatePeriod updatePeriod, LensDriver driver) throws LensException { double weight = driver.getConf().getDouble(UPDATE_PERIOD_WEIGHT_PREFIX + updatePeriod.name().toLowerCase(), updatePeriod.getNormalizationFactor()); return weight * updatePeriod.weight() / UpdatePeriod.DAILY.weight(); }
public static void filterPartitionsByUpdatePeriod(List<Partition> partitions, UpdatePeriod updatePeriod) { Iterator<Partition> iter = partitions.iterator(); while (iter.hasNext()) { Partition part = iter.next(); if (!UpdatePeriod.valueOf(part.getParameters().get(PARTITION_UPDATE_PERIOD)).equals(updatePeriod)) { iter.remove(); } } }
@Test(dataProvider = "update-periods") public void testFormat(UpdatePeriod period) throws Exception { RandomDateGenerator randomDateGenerator = new RandomDateGenerator(); for (int i = 0; i < 5000; i++) { Date randomDate = randomDateGenerator.nextDate(); randomDate = period.truncate(randomDate); assertEquals(randomDate, period.parse(period.format(randomDate))); } } }
private Set<FactPartition> getFactParts(List<String> partStrings) throws ParseException { Set<FactPartition> factParts = new HashSet<FactPartition>(); for (String partStr : partStrings) { String[] partEls = partStr.split(" "); UpdatePeriod p = null; String partSpec = partEls[1]; switch (partSpec.length()) { case 7: //monthly p = UpdatePeriod.MONTHLY; break; case 10: // daily p = UpdatePeriod.DAILY; break; case 13: // hourly p = UpdatePeriod.HOURLY; break; } FactPartition part = new FactPartition(partEls[0], p.parse(partSpec), p, null, p.format(), Collections.singleton("table1")); factParts.add(part); } return factParts; }
public static Map<String, String> getFullPartSpecAsMap(XPartition partition) { Map<String, String> spec = Maps.newHashMap(); if (partition.getTimePartitionSpec() != null) { for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) { spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format() .format(getDateFromXML(timePartSpecElement.getValue()))); } } if (partition.getNonTimePartitionSpec() != null) { for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) { spec.put(partSpecElement.getKey(), partSpecElement.getValue()); } } return spec; } }
public Calendar truncate(Calendar calendar) { Calendar cal = Calendar.getInstance(); cal.setTime(truncate(calendar.getTime())); return cal; }
public static TimePartition of(UpdatePeriod updatePeriod, String dateString) throws LensException { if (dateString == null || dateString.isEmpty()) { throw new LensException("time parition date string is null or blank"); } else { if (!updatePeriod.canParseDateString(dateString)) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString)); } try { return TimePartition.of(updatePeriod, updatePeriod.parse(dateString)); } catch (ParseException e) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString), e); } } }
public TimePartition partitionAtDiff(int increment) { Calendar cal = Calendar.getInstance(); cal.setTime(date); if (getUpdatePeriod().equals(UpdatePeriod.QUARTERLY)) { increment *= 3; } cal.add(updatePeriod.calendarField(), increment); return new TimePartition(updatePeriod, cal.getTime()); }
@Override protected SimpleDateFormat initialValue() { return new SimpleDateFormat(HOURLY.formatStr()); } };
/** extract update period from partition properties */ private UpdatePeriod deduceUpdatePeriod(Partition partition) { return UpdatePeriod.valueOf(partition.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD)); }
public static String getPartitionInfoKeyPrefix(UpdatePeriod updatePeriod, String partCol) { return STORAGE_PFX + PARTITION_TIMELINE_CACHE + updatePeriod.getName() + "." + partCol + "."; }
public void increment(Calendar calendar, int increment) { switch (this) { case QUARTERLY: increment *= 3; } calendar.add(calendarField(), increment); }
public static Date resolveRelativeDate(String str, Date now) throws LensException { if (StringUtils.isBlank(str)) { throw new LensException(LensCubeErrorCode.NULL_DATE_VALUE.getLensErrorInfo()); } // Resolve NOW with proper granularity Calendar calendar = Calendar.getInstance(); calendar.setTime(now); str = str.toLowerCase(); Matcher relativeMatcher = P_RELATIVE.matcher(str); if (relativeMatcher.find()) { String nowWithGranularity = relativeMatcher.group(); nowWithGranularity = nowWithGranularity.replaceAll("now", ""); nowWithGranularity = nowWithGranularity.replaceAll("\\.", ""); Matcher granularityMatcher = P_UNIT.matcher(nowWithGranularity); if (granularityMatcher.find()) { calendar = UpdatePeriod.fromUnitName(granularityMatcher.group().toLowerCase()).truncate(calendar); } } // Get rid of 'now' part and whitespace String diffStr = str.replaceAll(RELATIVE, "").replace(WSPACE, ""); TimeDiff diff = TimeDiff.parseFrom(diffStr); return diff.offsetFrom(calendar.getTime()); }
if (!nonTimeParts.containsKey(entry.getKey())) { try { updatePeriod.parse(entry.getValue()); } catch (java.text.ParseException e) { ignore = true;
/** * Truncates partitions in {@link #rangeToPartitions} such that only partitions belonging to * the passed undatePeriod are retained. * @param updatePeriod */ private void truncatePartitions(UpdatePeriod updatePeriod) { Iterator<Map.Entry<TimeRange, Set<FactPartition>>> rangeItr = rangeToPartitions.entrySet().iterator(); while (rangeItr.hasNext()) { Map.Entry<TimeRange, Set<FactPartition>> rangeEntry = rangeItr.next(); rangeEntry.getValue().removeIf(factPartition -> !factPartition.getPeriod().equals(updatePeriod)); rangeEntry.getValue().forEach(factPartition -> { factPartition.getStorageTables().remove(storageTable); factPartition.getStorageTables().add(resolvedName); }); if (rangeEntry.getValue().isEmpty()) { rangeItr.remove(); } } }
/** * Truncate time range using the update period. * The lower value of the truncated time range is the smallest date value equal to or larger than original * time range's lower value which lies at the update period's boundary. Similarly for higher value. * @param updatePeriod Update period to truncate time range with * @return truncated time range * @throws LensException If the truncated time range is invalid. */ public TimeRange truncate(UpdatePeriod updatePeriod) throws LensException { TimeRange timeRange = TimeRange.builder().partitionColumn(partitionColumn) .fromDate(updatePeriod.getCeilDate(fromDate)).toDate(updatePeriod.getFloorDate(toDate)).build(); timeRange.validate(); return timeRange; }