@Override public Iterable<String> apply(WindowedDataSegment dataSegment) { return dataSegment.getSegment().getDimensions(); } }
@VisibleForTesting static List<String> getUniqueDimensions( List<TimelineObjectHolder<String, DataSegment>> timelineSegments, @Nullable Set<String> excludeDimensions ) { final BiMap<String, Integer> uniqueDims = HashBiMap.create(); // Here, we try to retain the order of dimensions as they were specified since the order of dimensions may be // optimized for performance. // Dimensions are extracted from the recent segments to olders because recent segments are likely to be queried more // frequently, and thus the performance should be optimized for recent ones rather than old ones. // timelineSegments are sorted in order of interval int index = 0; for (TimelineObjectHolder<String, DataSegment> timelineHolder : Lists.reverse(timelineSegments)) { for (PartitionChunk<DataSegment> chunk : timelineHolder.getObject()) { for (String dimension : chunk.getObject().getDimensions()) { if (!uniqueDims.containsKey(dimension) && (excludeDimensions == null || !excludeDimensions.contains(dimension))) { uniqueDims.put(dimension, index++); } } } } final BiMap<Integer, String> orderedDims = uniqueDims.inverse(); return IntStream.range(0, orderedDims.size()) .mapToObj(orderedDims::get) .collect(Collectors.toList()); }
/** * Return a pair of maps containing the ranges of availability for dimensions and metrics respectively. * * @param metadata The metadata that contain segment availability. * * @return A pair of two maps of range sets. One entry is for dimensions and the other for metrics. */ public static Map<Columns, Map<String, RangeSet<DateTime>>> getRangeLists(DataSourceMetadata metadata) { final Map<String, RangeSet<DateTime>> dimensionIntervals = new HashMap<>(); final Map<String, RangeSet<DateTime>> metricIntervals = new HashMap<>(); metadata.segments.stream() .forEach( segment -> { buildRangeSet(segment.getDimensions(), segment.getInterval(), dimensionIntervals); buildRangeSet(segment.getMetrics(), segment.getInterval(), metricIntervals); } ); EnumMap<Columns, Map<String, RangeSet<DateTime>>> intervals = new EnumMap<>(Columns.class); intervals.put(DIMENSIONS, dimensionIntervals); intervals.put(METRICS, metricIntervals); return intervals; }
@GET @Path("/{dataSourceName}/dimensions") @Produces("application/json") public Iterable<String> getDatasourceDimensions( @PathParam("dataSourceName") String dataSourceName, @QueryParam("interval") String interval ) { final List<DataSegment> segments = getSegmentsForDatasources().get(dataSourceName); final Set<String> dims = Sets.newHashSet(); if (segments == null || segments.isEmpty()) { return dims; } Interval theInterval; if (interval == null || interval.isEmpty()) { DateTime now = new DateTime(); theInterval = new Interval(now.minusMillis(SEGMENT_HISTORY_MILLIS), now); } else { theInterval = new Interval(interval); } for (DataSegment segment : segments) { if (theInterval.overlaps(segment.getInterval())) { dims.addAll(segment.getDimensions()); } } return dims; }
@GET @Path("/{dataSourceName}/dimensions") @Produces(MediaType.APPLICATION_JSON) @ResourceFilters(DatasourceResourceFilter.class) public Iterable<String> getDatasourceDimensions( @PathParam("dataSourceName") String dataSourceName, @QueryParam("interval") String interval ) { final List<DataSegment> segments = getSegmentsForDatasources().get(dataSourceName); final Set<String> dims = Sets.newHashSet(); if (segments == null || segments.isEmpty()) { return dims; } Interval theInterval; if (interval == null || interval.isEmpty()) { DateTime now = getCurrentTime(); theInterval = new Interval(segmentMetadataQueryConfig.getDefaultHistory(), now); } else { theInterval = Intervals.of(interval); } for (DataSegment segment : segments) { if (theInterval.overlaps(segment.getInterval())) { dims.addAll(segment.getDimensions()); } } return dims; }
/** * Return a pair of maps containing the simplified intervals for dimensions and metrics respectively. * * @param metadata The metadata that contain segment availability. * * @return A pair of two maps of simplified intervals. One entry is for dimensions and the other for metrics. */ public static Map<Columns, Map<String, SimplifiedIntervalList>> getIntervalLists(DataSourceMetadata metadata) { Map<String, Set<Interval>> unsortedDimensionIntervals = new HashMap<>(); Map<String, Set<Interval>> unsortedMetricIntervals = new HashMap<>(); metadata.segments.stream() .forEach( segment -> { buildIntervalSet( segment.getDimensions(), segment.getInterval(), unsortedDimensionIntervals ); buildIntervalSet(segment.getMetrics(), segment.getInterval(), unsortedMetricIntervals); } ); EnumMap<Columns, Map<String, SimplifiedIntervalList>> intervals = new EnumMap<>(Columns.class); intervals.put(DIMENSIONS, toSimplifiedIntervalMap(unsortedDimensionIntervals)); intervals.put(METRICS, toSimplifiedIntervalMap(unsortedMetricIntervals)); return intervals; }
private static DataSegment computeMergedSegment( final String dataSource, final String version, final List<DataSegment> segments ) { final Interval mergedInterval = computeMergedInterval(segments); final Set<String> mergedDimensions = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); final Set<String> mergedMetrics = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (DataSegment segment : segments) { mergedDimensions.addAll(segment.getDimensions()); mergedMetrics.addAll(segment.getMetrics()); } return DataSegment.builder() .dataSource(dataSource) .interval(mergedInterval) .version(version) .binaryVersion(IndexIO.CURRENT_VERSION_ID) .shardSpec(NoneShardSpec.instance()) .dimensions(Lists.newArrayList(mergedDimensions)) .metrics(Lists.newArrayList(mergedMetrics)) .build(); } }
private static DataSegment computeMergedSegment( final String dataSource, final String version, final List<DataSegment> segments ) { final Interval mergedInterval = computeMergedInterval(segments); final Set<String> mergedDimensions = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); final Set<String> mergedMetrics = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (DataSegment segment : segments) { mergedDimensions.addAll(segment.getDimensions()); mergedMetrics.addAll(segment.getMetrics()); } return DataSegment.builder() .dataSource(dataSource) .interval(mergedInterval) .version(version) .binaryVersion(IndexIO.CURRENT_VERSION_ID) .shardSpec(new NoneShardSpec()) .dimensions(Lists.newArrayList(mergedDimensions)) .metrics(Lists.newArrayList(mergedMetrics)) .build(); } }
for (ServerSelector server : partitionHolder.payloads()) { final DataSegment segment = server.getSegment(); dimensions.addAll(segment.getDimensions()); metrics.addAll(segment.getMetrics());
/** * Given a druid data segment constructs an object to hold the information of this partition. * * @param segment The druid data segments that corresponds to a specific partition of a druid segment. */ public SegmentInfo(DataSegment segment) { this.dataSource = segment.getDataSource(); this.interval = segment.getInterval(); this.dimensions = segment.getDimensions(); this.metrics = segment.getMetrics(); this.version = segment.getVersion(); ShardSpec spec = segment.getShardSpec(); this.shardSpec = spec instanceof NumberedShardSpec ? (NumberedShardSpec) spec : new NumberedShardSpec((NoneShardSpec) spec); this.size = segment.getSize(); this.identifier = segment.getIdentifier(); }