public BitmapCreationBenchmark(Class<? extends BitmapSerdeFactory> clazz) throws IllegalAccessException, InstantiationException { BitmapSerdeFactory serdeFactory = clazz.newInstance(); factory = serdeFactory.getBitmapFactory(); }
@Override public ColumnValueSelector<?> makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap()); }
@Override public ColumnValueSelector<?> makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap()); }
@Override public ColumnValueSelector<?> makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap()); }
public StringDimensionMergerV9( String dimensionName, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, ProgressIndicator progress, Closer closer ) { this.dimensionName = dimensionName; this.indexSpec = indexSpec; this.capabilities = capabilities; this.segmentWriteOutMedium = segmentWriteOutMedium; nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); this.progress = progress; this.closer = closer; }
@Override public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( buffer, byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } builder.setType(ValueType.LONG) .setHasMultipleValues(false) .setNumericColumnSupplier(new LongNumericColumnSupplier(column, bitmap)); }; } }
@Override public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); final CompressedColumnarFloatsSupplier column = CompressedColumnarFloatsSupplier.fromByteBuffer( buffer, byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) .setNumericColumnSupplier(new FloatNumericColumnSupplier(column, bitmap)); }; } }
@Override public long getSerializedSize() throws IOException { nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); long bitmapSize = nullRowsBitmap.isEmpty() ? 0L : nullValueBitmapWriter.getSerializedSize(); return Integer.BYTES + writer.getSerializedSize() + bitmapSize; }
@Override public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); final Supplier<ColumnarDoubles> column = CompressedColumnarDoublesSuppliers.fromByteBuffer( buffer, byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } builder.setType(ValueType.DOUBLE) .setHasMultipleValues(false) .setNumericColumnSupplier(new DoubleNumericColumnSupplier(column, bitmap)); }; } }
@Override public long getSerializedSize() throws IOException { nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); long bitmapSize = nullRowsBitmap.isEmpty() ? 0L : nullValueBitmapWriter.getSerializedSize(); return Integer.BYTES + writer.getSerializedSize() + bitmapSize; }
@Override public long getSerializedSize() throws IOException { nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); long bitmapSize = nullRowsBitmap.isEmpty() ? 0L : nullValueBitmapWriter.getSerializedSize(); return Integer.BYTES + writer.getSerializedSize() + bitmapSize; }
@Override public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) { final Supplier<ColumnarDoubles> column = CompressedColumnarDoublesSuppliers.fromByteBuffer( buffer, byteOrder ); DoubleNumericColumnSupplier columnSupplier = new DoubleNumericColumnSupplier( column, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() ); builder.setType(ValueType.DOUBLE) .setHasMultipleValues(false) .setNumericColumnSupplier(columnSupplier); } };
@Override public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) { final CompressedColumnarFloatsSupplier column = CompressedColumnarFloatsSupplier.fromByteBuffer( buffer, byteOrder ); FloatNumericColumnSupplier columnSupplier = new FloatNumericColumnSupplier( column, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() ); builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) .setNumericColumnSupplier(columnSupplier); } };
@Override public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) { final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( buffer, byteOrder ); LongNumericColumnSupplier columnSupplier = new LongNumericColumnSupplier( column, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) .setNumericColumnSupplier(columnSupplier); } };
@Override public void open() throws IOException { writer = CompressionFactory.getDoubleSerializer( segmentWriteOutMedium, StringUtils.format("%s.double_column", filenameBase), byteOrder, compression ); writer.open(); nullValueBitmapWriter = new ByteBufferWriter<>( segmentWriteOutMedium, bitmapSerdeFactory.getObjectStrategy() ); nullValueBitmapWriter.open(); nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); }
@Override public void open() throws IOException { writer = CompressionFactory.getFloatSerializer( segmentWriteOutMedium, StringUtils.format("%s.float_column", filenameBase), byteOrder, compression ); writer.open(); nullValueBitmapWriter = new ByteBufferWriter<>( segmentWriteOutMedium, bitmapSerdeFactory.getObjectStrategy() ); nullValueBitmapWriter.open(); nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); }
@Override public void open() throws IOException { writer = CompressionFactory.getLongSerializer( segmentWriteOutMedium, StringUtils.format("%s.long_column", filenameBase), byteOrder, encoding, compression ); writer.open(); nullValueBitmapWriter = new ByteBufferWriter<>( segmentWriteOutMedium, bitmapSerdeFactory.getObjectStrategy() ); nullValueBitmapWriter.open(); nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); }
@Before public void setUp() throws IndexSizeExceededException { long timestamp = 0L; for (Map<String, Object> event : events1) { incrementalIndex1.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event)); } timestamp = 0L; for (Map<String, Object> event : events2) { incrementalIndex2.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event)); } adapter2 = new IncrementalIndexAdapter( DEFAULT_INTERVAL, incrementalIndex2, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory() ); adapter1 = new IncrementalIndexAdapter( DEFAULT_INTERVAL, incrementalIndex1, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory() ); }
@Test public void testGetRowsIterable() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex toPersist1 = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, toPersist1); final IndexableAdapter incrementalAdapter = new IncrementalIndexAdapter( toPersist1.getInterval(), toPersist1, INDEX_SPEC.getBitmapSerdeFactory() .getBitmapFactory() ); RowIterator rows = incrementalAdapter.getRows(); List<Integer> rowNums = new ArrayList<>(); while (rows.moveToNext()) { rowNums.add(rows.getPointer().getRowNum()); } Assert.assertEquals(2, rowNums.size()); Assert.assertEquals(0, (long) rowNums.get(0)); Assert.assertEquals(1, (long) rowNums.get(1)); }
@Test public void testGetBitmapIndex() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex incrementalIndex = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, incrementalIndex); IndexableAdapter adapter = new IncrementalIndexAdapter( incrementalIndex.getInterval(), incrementalIndex, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory() ); String dimension = "dim1"; try (CloseableIndexed<String> dimValueLookup = adapter.getDimValueLookup(dimension)) { for (int i = 0; i < dimValueLookup.size(); i++) { BitmapValues bitmapValues = adapter.getBitmapValues(dimension, i); Assert.assertEquals(1, bitmapValues.size()); } } }