@Override public DimensionMergerV9 makeMerger( IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, ProgressIndicator progress, Closer closer ) { // Sanity-check capabilities. if (hasBitmapIndexes != capabilities.hasBitmapIndexes()) { throw new ISE( "capabilities.hasBitmapIndexes[%s] != this.hasBitmapIndexes[%s]", capabilities.hasBitmapIndexes(), hasBitmapIndexes ); } return new StringDimensionMergerV9(dimensionName, indexSpec, segmentWriteOutMedium, capabilities, progress, closer); } }
@Override @Nullable public Comparable getMinValue(String dimension) { ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null; } return null; }
/** * Split the given dimensions list into bitmap-supporting dimensions and non-bitmap supporting ones. * Note that the returned lists are free to modify. */ private static Pair<List<DimensionSpec>, List<DimensionSpec>> partitionDimensionList( StorageAdapter adapter, List<DimensionSpec> dimensions ) { final List<DimensionSpec> bitmapDims = new ArrayList<>(); final List<DimensionSpec> nonBitmapDims = new ArrayList<>(); final List<DimensionSpec> dimsToSearch = getDimsToSearch( adapter.getAvailableDimensions(), dimensions ); for (DimensionSpec spec : dimsToSearch) { ColumnCapabilities capabilities = adapter.getColumnCapabilities(spec.getDimension()); if (capabilities == null) { continue; } if (capabilities.hasBitmapIndexes()) { bitmapDims.add(spec); } else { nonBitmapDims.add(spec); } } return new Pair<>(bitmapDims, nonBitmapDims); }
@Override @Nullable public Comparable getMaxValue(String dimension) { ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null; } return null; }
dimension, dimensionHandler.getMultivalueHandling(), columnHolder.getCapabilities().hasBitmapIndexes()
@Override public BitmapValues getBitmapValues(String dimension, int index) { DimensionAccessor accessor = accessors.get(dimension); if (accessor == null) { return BitmapValues.EMPTY; } ColumnCapabilities capabilities = accessor.dimensionDesc.getCapabilities(); DimensionIndexer indexer = accessor.dimensionDesc.getIndexer(); if (!capabilities.hasBitmapIndexes()) { return BitmapValues.EMPTY; } final int id = (Integer) indexer.getUnsortedEncodedValueFromSorted(index); if (id < 0 || id >= indexer.getCardinality()) { return BitmapValues.EMPTY; } MutableBitmap bitmapIndex = accessor.invertedIndexes[id]; if (bitmapIndex == null) { return BitmapValues.EMPTY; } return new MutableBitmapValues(bitmapIndex); }
if (capabilities.hasBitmapIndexes()) { final MutableBitmap[] bitmapIndexes = accessor.invertedIndexes; final DimensionIndexer indexer = accessor.indexer;
} else if (columnHolder.getCapabilities().hasBitmapIndexes()) { return columnHolder.getBitmapIndex(); } else {
public void merge(ColumnCapabilities other) { if (other == null) { return; } if (type == null) { type = other.getType(); } if (!type.equals(other.getType())) { throw new ISE("Cannot merge columns of type[%s] and [%s]", type, other.getType()); } this.dictionaryEncoded |= other.isDictionaryEncoded(); this.runLengthEncoded |= other.isRunLengthEncoded(); this.hasInvertedIndexes |= other.hasBitmapIndexes(); this.hasSpatialIndexes |= other.hasSpatialIndexes(); this.hasMultipleValues |= other.hasMultipleValues(); this.filterable &= other.isFilterable(); } }
@Override public ImmutableBitmap getBitmapIndex(String dimension, String value) { if (isVirtualColumn(dimension)) { // Virtual columns don't have dictionaries or indexes. return null; } final ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null || !columnHolder.getCapabilities().isFilterable()) { if (NullHandling.isNullOrEquivalent(value)) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); } else { return bitmapFactory.makeEmptyImmutableBitmap(); } } if (!columnHolder.getCapabilities().hasBitmapIndexes()) { return null; } final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); return bitmapIndex.getBitmap(bitmapIndex.getIndex(value)); }
Assert.assertTrue(adapter.getCapabilities("dimA").hasBitmapIndexes()); checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimA", "1")); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimB").hasBitmapIndexes());
); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim1").hasBitmapIndexes()); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); ); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim1").hasBitmapIndexes()); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); ); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim1").hasBitmapIndexes()); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes());
@Override public void writeIndexes(@Nullable List<IntBuffer> segmentRowNumConversions) throws IOException if (!capabilities.hasBitmapIndexes()) { return;
public static DimensionHandler getHandlerFromCapabilities( String dimensionName, ColumnCapabilities capabilities, MultiValueHandling multiValueHandling ) { if (capabilities == null) { return new StringDimensionHandler(dimensionName, multiValueHandling, true); } multiValueHandling = multiValueHandling == null ? MultiValueHandling.ofDefault() : multiValueHandling; if (capabilities.getType() == ValueType.STRING) { if (!capabilities.isDictionaryEncoded()) { throw new IAE("String column must have dictionary encoding."); } return new StringDimensionHandler(dimensionName, multiValueHandling, capabilities.hasBitmapIndexes()); } if (capabilities.getType() == ValueType.LONG) { return new LongDimensionHandler(dimensionName); } if (capabilities.getType() == ValueType.FLOAT) { return new FloatDimensionHandler(dimensionName); } if (capabilities.getType() == ValueType.DOUBLE) { return new DoubleDimensionHandler(dimensionName); } // Return a StringDimensionHandler by default (null columns will be treated as String typed) return new StringDimensionHandler(dimensionName, multiValueHandling, true); }
Assert.assertEquals(Collections.singletonList(2L), rowList.get(3).metricValues()); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimA").hasBitmapIndexes()); Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimC").hasBitmapIndexes());
@Test public void testSerde() throws Exception { String json = mapper.writeValueAsString(new ColumnCapabilitiesImpl() .setDictionaryEncoded(true) .setHasBitmapIndexes(true) .setHasMultipleValues(true) .setHasSpatialIndexes(true) .setType(ValueType.COMPLEX) .setFilterable(true)); Assert.assertFalse(json.contains("filterable")); ColumnCapabilities cc = mapper.readValue(json, ColumnCapabilitiesImpl.class); Assert.assertEquals(ValueType.COMPLEX, cc.getType()); Assert.assertTrue(cc.isDictionaryEncoded()); Assert.assertFalse(cc.isRunLengthEncoded()); Assert.assertTrue(cc.hasSpatialIndexes()); Assert.assertTrue(cc.hasMultipleValues()); Assert.assertTrue(cc.hasBitmapIndexes()); Assert.assertFalse(cc.isFilterable()); }
@Test public void testDeserialization() throws Exception { String json = "{\n" + " \"type\":\"COMPLEX\",\n" + " \"dictionaryEncoded\":true,\n" + " \"runLengthEncoded\":true,\n" + " \"hasSpatialIndexes\":true,\n" + " \"hasMultipleValues\":true,\n" + " \"hasBitmapIndexes\":true,\n" + " \"filterable\":true\n" + "}"; ColumnCapabilities cc = mapper.readValue(json, ColumnCapabilitiesImpl.class); Assert.assertEquals(ValueType.COMPLEX, cc.getType()); Assert.assertTrue(cc.isDictionaryEncoded()); Assert.assertTrue(cc.isRunLengthEncoded()); Assert.assertTrue(cc.hasSpatialIndexes()); Assert.assertTrue(cc.hasMultipleValues()); Assert.assertTrue(cc.hasBitmapIndexes()); Assert.assertFalse(cc.isFilterable()); } }
Comparable max = null; if (!capabilities.hasBitmapIndexes()) { return ColumnAnalysis.error("string_no_bitmap");
@Override @Nullable public Comparable getMinValue(String dimension) { ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null; } return null; }
@Override @Nullable public Comparable getMaxValue(String dimension) { ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null; } return null; }