private boolean canBePurged(FileStatus f, DateUtil.TimeDiff retention) { return f.getModificationTime() < retention.negativeOffsetFrom(Calendar.getInstance().getTime()).getTime(); }
public void init(Configuration configuration) { this.conf = configuration; this.resultsetPath = new Path(conf.get(RESULT_SET_PARENT_DIR, RESULT_SET_PARENT_DIR_DEFAULT)); this.hdfsOutputPath = new Path(resultsetPath.toString(), conf.get(QUERY_HDFS_OUTPUT_PATH, DEFAULT_HDFS_OUTPUT_PATH)); int purgeDelay = conf.getInt(RESULTSET_PURGE_INTERVAL_IN_SECONDS, DEFAULT_RESULTSET_PURGE_INTERVAL_IN_SECONDS); try { String resultSetDiffStr = conf.get(QUERY_RESULTSET_RETENTION, DEFAULT_QUERY_RESULTSET_RETENTION); String hdfsOutputDiffStr = conf.get(QUERY_RESULTSET_RETENTION, DEFAULT_QUERY_RESULTSET_RETENTION); this.resultsetRetention = DateUtil.TimeDiff.parseFrom(resultSetDiffStr); this.hdfsOutputRetention = DateUtil.TimeDiff.parseFrom(hdfsOutputDiffStr); queryResultPurgerExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { @Override public Thread newThread(Runnable r) { return new Thread(r, "QueryResultPurger"); } }); queryResultPurgerExecutor.scheduleWithFixedDelay(this, purgeDelay, purgeDelay, TimeUnit.SECONDS); log.info( "Initialized query result purger with lens resultset retention of {} and hdfs output retention of {}, " + "scheduled to run every {} seconds", resultSetDiffStr, hdfsOutputDiffStr, purgeDelay); } catch (LensException e) { log.error("Error occurred while initializing query result purger", e); } }
public static Date resolveRelativeDate(String str, Date now) throws LensException { if (StringUtils.isBlank(str)) { throw new LensException(LensCubeErrorCode.NULL_DATE_VALUE.getLensErrorInfo()); } // Resolve NOW with proper granularity Calendar calendar = Calendar.getInstance(); calendar.setTime(now); str = str.toLowerCase(); Matcher relativeMatcher = P_RELATIVE.matcher(str); if (relativeMatcher.find()) { String nowWithGranularity = relativeMatcher.group(); nowWithGranularity = nowWithGranularity.replaceAll("now", ""); nowWithGranularity = nowWithGranularity.replaceAll("\\.", ""); Matcher granularityMatcher = P_UNIT.matcher(nowWithGranularity); if (granularityMatcher.find()) { calendar = UpdatePeriod.fromUnitName(granularityMatcher.group().toLowerCase()).truncate(calendar); } } // Get rid of 'now' part and whitespace String diffStr = str.replaceAll(RELATIVE, "").replace(WSPACE, ""); TimeDiff diff = TimeDiff.parseFrom(diffStr); return diff.offsetFrom(calendar.getTime()); }
return null; DateUtil.TimeDiff diff1 = DateUtil.TimeDiff.parseFrom(matcher.group(2).trim()); DateUtil.TimeDiff diff2 = DateUtil.TimeDiff.parseFrom(matcher.group(3).trim()); String relatedTimeDim = matcher.group(1).trim(); String fallbackPartCol = baseCube.getPartitionColumnOfTimeDim(relatedTimeDim); return TimeRange.builder().fromDate(diff2.negativeOffsetFrom(range.getFromDate())) .toDate(diff1.negativeOffsetFrom(range.getToDate())).partitionColumn(fallbackPartCol).build();
return null; DateUtil.TimeDiff diff1 = DateUtil.TimeDiff.parseFrom(matcher.group(2).trim()); DateUtil.TimeDiff diff2 = DateUtil.TimeDiff.parseFrom(matcher.group(3).trim()); String relatedTimeDim = matcher.group(1).trim(); String fallbackPartCol = baseCube.getPartitionColumnOfTimeDim(relatedTimeDim); return TimeRange.builder().fromDate(diff2.negativeOffsetFrom(range.getFromDate())) .toDate(diff1.negativeOffsetFrom(range.getToDate())).partitionColumn(fallbackPartCol).build();
@Test public void testTimeDiff() throws LensException { ArrayList<String> minusFourDays = Lists.newArrayList("-4 days", "-4days", "-4day", "-4 day", "- 4days", "- 4 day"); ArrayList<String> plusFourDays = Lists.newArrayList("+4 days", "4 days", "+4days", "4day", "4 day", "+ 4days", "+ 4 day", "+4 day"); Set<TimeDiff> diffs = Sets.newHashSet(); for (String diffStr : minusFourDays) { diffs.add(TimeDiff.parseFrom(diffStr)); } assertEquals(diffs.size(), 1); TimeDiff minusFourDaysDiff = diffs.iterator().next(); assertEquals(minusFourDaysDiff.quantity, -4); assertEquals(minusFourDaysDiff.updatePeriod, DAILY); diffs.clear(); for (String diffStr : plusFourDays) { diffs.add(TimeDiff.parseFrom(diffStr)); } assertEquals(diffs.size(), 1); TimeDiff plusFourDaysDiff = diffs.iterator().next(); assertEquals(plusFourDaysDiff.quantity, 4); assertEquals(plusFourDaysDiff.updatePeriod, DAILY); Date now = new Date(); assertEquals(minusFourDaysDiff.offsetFrom(plusFourDaysDiff.offsetFrom(now)), now); assertEquals(plusFourDaysDiff.offsetFrom(minusFourDaysDiff.offsetFrom(now)), now); assertEquals(minusFourDaysDiff.negativeOffsetFrom(now), plusFourDaysDiff.offsetFrom(now)); assertEquals(minusFourDaysDiff.offsetFrom(now), plusFourDaysDiff.negativeOffsetFrom(now)); }
public static Date resolveRelativeDate(String str, Date now) throws LensException { if (StringUtils.isBlank(str)) { throw new LensException(LensCubeErrorCode.NULL_DATE_VALUE.getLensErrorInfo()); } // Resolve NOW with proper granularity Calendar calendar = Calendar.getInstance(); calendar.setTime(now); str = str.toLowerCase(); Matcher relativeMatcher = P_RELATIVE.matcher(str); if (relativeMatcher.find()) { String nowWithGranularity = relativeMatcher.group(); nowWithGranularity = nowWithGranularity.replaceAll("now", ""); nowWithGranularity = nowWithGranularity.replaceAll("\\.", ""); Matcher granularityMatcher = P_UNIT.matcher(nowWithGranularity); if (granularityMatcher.find()) { calendar = UpdatePeriod.fromUnitName(granularityMatcher.group().toLowerCase()).truncate(calendar); } } // Get rid of 'now' part and whitespace String diffStr = str.replaceAll(RELATIVE, "").replace(WSPACE, ""); TimeDiff diff = TimeDiff.parseFrom(diffStr); return diff.offsetFrom(calendar.getTime()); }
public static TimeDiff parseFrom(String diffStr) throws LensException { // Get the relative diff part to get eventual date based on now. Matcher qtyMatcher = P_QUANTITY.matcher(diffStr); int qty = 1; if (qtyMatcher.find()) { qty = Integer.parseInt(qtyMatcher.group()); } Matcher signageMatcher = P_SIGNAGE.matcher(diffStr); if (signageMatcher.find()) { String sign = signageMatcher.group(); if ("-".equals(sign)) { qty = -qty; } } Matcher unitMatcher = P_UNIT.matcher(diffStr); if (unitMatcher.find()) { return new TimeDiff(qty, UpdatePeriod.fromUnitName(unitMatcher.group().toLowerCase())); } return new TimeDiff(0, UpdatePeriod.CONTINUOUS); }
public static TimeDiff parseFrom(String diffStr) throws LensException { // Get the relative diff part to get eventual date based on now. Matcher qtyMatcher = P_QUANTITY.matcher(diffStr); int qty = 1; if (qtyMatcher.find()) { qty = Integer.parseInt(qtyMatcher.group()); } Matcher signageMatcher = P_SIGNAGE.matcher(diffStr); if (signageMatcher.find()) { String sign = signageMatcher.group(); if ("-".equals(sign)) { qty = -qty; } } Matcher unitMatcher = P_UNIT.matcher(diffStr); if (unitMatcher.find()) { return new TimeDiff(qty, UpdatePeriod.fromUnitName(unitMatcher.group().toLowerCase())); } return new TimeDiff(0, UpdatePeriod.CONTINUOUS); }