@Override public SimplifiedIntervalList getVolatileIntervals() { return new SimplifiedIntervalList(); } }
/** * Return the volatile intervals from the context. * * @param context The RequestContext object containing the volatile intervals if any * * @return the volatile intervals from the request or an empty list */ public static SimplifiedIntervalList getVolatileIntervalsWithDefault(Map<String, Serializable> context) { return new SimplifiedIntervalList( (Collection<Interval>) context.computeIfAbsent( VOLATILE_INTERVALS_CONTEXT_KEY.getName(), (ignored) -> new SimplifiedIntervalList() ) ); } }
/** * Return the missing intervals from the context. * <p> * <b>WARNING</b>: A serialization issue may result in the context value being a list but not a * Simplified Interval List. See https://github.com/yahoo/fili/issues/657 * * @param context The map containing the missing intervals if any * * @return the missing intervals from the request or an empty list */ public static SimplifiedIntervalList getPartialIntervalsWithDefault(Map<String, Serializable> context) { return new SimplifiedIntervalList( (Collection<Interval>) context.computeIfAbsent( MISSING_INTERVALS_CONTEXT_KEY.getName(), (ignored) -> new SimplifiedIntervalList() ) ); } }
/** * Return a view of the available intervals. * * @return The widest set of intervals that the table can report on */ default SimplifiedIntervalList getAvailableIntervals() { // By default union all available columns return getAllAvailableIntervals().values().stream() .reduce(SimplifiedIntervalList::union) .orElse(new SimplifiedIntervalList()); }
/** * Fetch a {@link SimplifiedIntervalList} representing the coalesced available intervals on this availability. * * @return A <tt>SimplifiedIntervalList</tt> of intervals available */ default SimplifiedIntervalList getAvailableIntervals() { return getAllAvailableIntervals().values().stream() .reduce(SimplifiedIntervalList::union) .orElse(new SimplifiedIntervalList()); }
@Override public SimplifiedIntervalList getAvailableIntervals(PhysicalDataSourceConstraint constraint) { Set<String> requestColumns = constraint.getAllColumnPhysicalNames(); if (requestColumns.isEmpty()) { return new SimplifiedIntervalList(); } // Need to ensure requestColumns is not empty in order to prevent returning null by reduce operation return requestColumns.stream() .map(physicalName -> getAllAvailableIntervals().getOrDefault( physicalName, new SimplifiedIntervalList() )) .reduce(SimplifiedIntervalList::intersect) .orElse(new SimplifiedIntervalList()); }
/** * Convert the map of unsorted intervals into a map of SimplifiedIntervalLists. * * @param unsortedIntervals Map to clean up * * @return the cleaned up map */ private static Map<String, SimplifiedIntervalList> toSimplifiedIntervalMap( Map<String, Set<Interval>> unsortedIntervals ) { return unsortedIntervals.entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, entry -> new SimplifiedIntervalList(entry.getValue()) ) ); }
/** * Sum the length of the intervals in this collection. * * @param intervals A collection of time intervals * * @return The total duration of all the intervals */ public static long getTotalDuration(Collection<Interval> intervals) { return new SimplifiedIntervalList(intervals).stream() .map(Interval::toDuration) .mapToLong(Duration::getMillis) .sum(); }
@Override public SimplifiedIntervalList getAvailableIntervals(PhysicalDataSourceConstraint constraint) { Set<String> dataSourceMetricNames = availabilitiesToMetricNames.values().stream() .flatMap(Set::stream) .collect(Collectors.toSet()); // If the table is configured with a column that is not supported by the underlying data sources if (!constraint.getMetricNames().stream().allMatch(dataSourceMetricNames::contains)) { return new SimplifiedIntervalList(); } return constructSubConstraint(constraint).entrySet().stream() .map(entry -> entry.getKey().getAvailableIntervals(entry.getValue())) .reduce(SimplifiedIntervalList::intersect).orElse(new SimplifiedIntervalList()); }
/** * Build a SegmentIntervalsHashIdGenerator that uses the raw simplified intervals of a druidAggregationQuery to * create a segmentId. * * @param dataSourceMetadataService A Service to get information about segment metadata */ public SegmentIntervalsHashIdGenerator(DataSourceMetadataService dataSourceMetadataService) { this( dataSourceMetadataService, new DefaultingDictionary<>( druidAggregationQuery -> new SimplifiedIntervalList(druidAggregationQuery.getIntervals()) ) ); }
/** * Returns union of availabilities of the logical table. * * @param logicalTable The logical table * * @return the union of availabilities of the logical table */ public static SimplifiedIntervalList logicalTableAvailability(LogicalTable logicalTable) { return logicalTable.getTableGroup().getPhysicalTables().stream() .map(PhysicalTable::getAllAvailableIntervals) .map(Map::entrySet) .flatMap(Set::stream) .map(Map.Entry::getValue) .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union); } }
/** * Build an iterator that will iterate over a set of intervals, collecting them into a single stream, and slicing * or not slicing them according to the granularity given. * <p> * IMPORTANT WARNING: The results of this iterator will not always align with the bucketing of query results! * <p> * Period based granularities will return a single interval per result time bucket from the query. For the 'all' * time grain, multiple intervals are possible, but in query result data they will all map to a single result * time bucket. * * @param intervals The intervals being iterated across * * @return A grain delimited iterator returning all the intervals in the interval set */ default Iterator<Interval> intervalsIterator(Collection<Interval> intervals) { return intervalsIterator(intervals instanceof SimplifiedIntervalList ? (SimplifiedIntervalList) intervals : new SimplifiedIntervalList(intervals) ); }
/** * Build a Map of Class to Function that should be used to get requestedIntervals from the DruidQuery. * * @return A Map that maps Class to a function that computes the requested intervals for a Druid query of * that particular Class */ protected Map<Class, RequestedIntervalsFunction> buildSigningFunctions() { DefaultingDictionary<Class, RequestedIntervalsFunction> signingFunctions = new DefaultingDictionary<>( druidAggregationQuery -> new SimplifiedIntervalList(druidAggregationQuery.getIntervals()) ); signingFunctions.put(LookbackQuery.class, new LookbackQuery.LookbackQueryRequestedIntervalsFunction()); return signingFunctions; }
/** * Returns union of constrained availabilities of constrained logical table. * * @param logicalTable The constrained logical table * @param queryPlanningConstraint The constraint * * @return the union of constrained availabilities of constrained logical table */ public static SimplifiedIntervalList getConstrainedLogicalTableAvailability( LogicalTable logicalTable, QueryPlanningConstraint queryPlanningConstraint ) { return logicalTable.getTableGroup().getPhysicalTables().stream() .map(physicalTable -> physicalTable.withConstraint(queryPlanningConstraint)) .map(PhysicalTable::getAvailableIntervals) .reduce(new SimplifiedIntervalList(), SimplifiedIntervalList::union); }
@Override public SimplifiedIntervalList getVolatileIntervals() { long now = currentTimeMillis.getAsLong(); return new SimplifiedIntervalList(Collections.singletonList(new Interval(now - past, now + future))); } }
@Override public Map<String, SimplifiedIntervalList> getAvailableIntervalsByDataSource(DataSourceName dataSourceName) { return testAvailableIntervals.entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, entry -> new SimplifiedIntervalList(entry.getValue()) ) ); } }
@Override public Map<String, SimplifiedIntervalList> getAvailableIntervalsByDataSource(DataSourceName dataSourceName) { return testAvailableIntervals.entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, entry -> new SimplifiedIntervalList(entry.getValue()) ) ); } }
/** * Calculate the missing intervals based on the subpartitions availability. * formula is: * Union(subpart availabilitiy) - Union(subpart missing intervals) * * @param constraint The filtering constraint * * @return The intervals which are available for the given constraint */ private SimplifiedIntervalList mergeAvailabilities(PhysicalDataSourceConstraint constraint) { SimplifiedIntervalList unionOfAvailableIntervals = new SimplifiedIntervalList(); SimplifiedIntervalList unionOfMissingIntervals = new SimplifiedIntervalList(); for (Availability availability : filteredAvailabilities(constraint).collect(Collectors.toSet())) { unionOfAvailableIntervals = unionOfAvailableIntervals.union(availability.getAvailableIntervals(constraint)); unionOfMissingIntervals = unionOfMissingIntervals.union( getBoundedMissingIntervalsWithConstraint(availability, constraint) ); } return unionOfAvailableIntervals.subtract(unionOfMissingIntervals); }
@Override public SimplifiedIntervalList getVolatileIntervals( Granularity granularity, List<Interval> intervals, PhysicalTable factSource ) { SimplifiedIntervalList simplifiedIntervals = new SimplifiedIntervalList(intervals); SimplifiedIntervalList volatileIntervals = IntervalUtils.collectBucketedIntervalsIntersectingIntervalList( intervalsFunctions.getOrDefault(factSource.getName(), defaultIntervals).getVolatileIntervals(), simplifiedIntervals, granularity ); if (granularity instanceof AllGranularity && !volatileIntervals.isEmpty()) { volatileIntervals = simplifiedIntervals; } LOG.trace("Volatile intervals: {} for grain {}", volatileIntervals, granularity); return volatileIntervals; } }
/** * Calculates the MISSING intervals for a given a availability and datasource constraint. * * @param availability The availability to find the missing intervals for * @param constraint The constraint for the query * @return the missing intervals */ private SimplifiedIntervalList getBoundedMissingIntervalsWithConstraint( Availability availability, PhysicalDataSourceConstraint constraint ) { SimplifiedIntervalList availableIntervals = availability.getAvailableIntervals(constraint); DateTime expectedStart = availability.getExpectedStartDate(constraint).orElse(Availability.DISTANT_PAST); DateTime expectedEnd = availability.getExpectedEndDate(constraint).orElse(Availability.FAR_FUTURE); return new SimplifiedIntervalList(Collections.singleton(new Interval(expectedStart, expectedEnd))) .subtract(availableIntervals); }