@Override public BitSet getNullBits() { return columnPage.getNullBits(); }
private byte[] getNullBitSet(int rowId, ColumnType columnType) { if (columnPage.getNullBits().get(rowId) && columnType == ColumnType.COMPLEX_PRIMITIVE) { // if this row is null, return default null represent in byte array return CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; } if (columnPage.getNullBits().get(rowId)) { // if this row is null, return default null represent in byte array return CarbonCommonConstants.EMPTY_BYTE_ARRAY; } return null; }
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); Object[] filterValues = this.msrFilterRangeValues; DataType msrType = msrColEvalutorInfoList.get(0).getType(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); BitSet nullBitSet = columnPage.getNullBits(); for (int i = 0; i < filterValues.length; i++) { if (filterValues[i] == null) { for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { if (!nullBitSet.get(startIndex)) { Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(columnPage, startIndex, msrType, msrColEvalutorInfoList.get(0).getMeasure()); if (comparator.compare(msrValue, filterValues[i]) > 0) { // This is a match. bitSet.set(startIndex); } } } } return bitSet; }
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); Object[] filterValues = this.msrFilterRangeValues; DataType msrType = msrColEvalutorInfoList.get(0).getType(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrType); BitSet nullBitSet = columnPage.getNullBits(); for (int i = 0; i < filterValues.length; i++) { if (filterValues[i] == null) { for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { if (!nullBitSet.get(startIndex)) { Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(columnPage, startIndex, msrType, msrColEvalutorInfoList.get(0).getMeasure()); if (comparator.compare(msrValue, filterValues[i]) < 0) { // This is a match. bitSet.set(startIndex); } } } } return bitSet; }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putLong(vectorOffset, dataChunk.getLong(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putLong(vectorOffset, dataChunk.getLong(i)); } vectorOffset++; } } }
@Override public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; vector.putShort(vectorOffset, (short) dataChunk.getLong(currentRow)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; if (nullBitSet.get(currentRow)) { vector.putNull(vectorOffset); } else { vector.putShort(vectorOffset, (short) dataChunk.getLong(currentRow)); } vectorOffset++; } } } }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putInt(vectorOffset, (int)dataChunk.getLong(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putInt(vectorOffset, (int)dataChunk.getLong(i)); } vectorOffset++; } } }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putBoolean(vectorOffset, dataChunk.getBoolean(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putBoolean(vectorOffset, dataChunk.getBoolean(i)); } vectorOffset++; } } }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putDouble(vectorOffset, dataChunk.getDouble(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putDouble(vectorOffset, dataChunk.getDouble(i)); } vectorOffset++; } } }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putShort(vectorOffset, (short) dataChunk.getLong(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putShort(vectorOffset, (short) dataChunk.getLong(i)); } vectorOffset++; } } }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putFloat(vectorOffset, dataChunk.getFloat(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putFloat(vectorOffset, dataChunk.getFloat(i)); } vectorOffset++; } } }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { vector.putByte(vectorOffset, dataChunk.getByte(i)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { vector.putByte(vectorOffset, dataChunk.getByte(i)); } vectorOffset++; } } }
@Override public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; if (nullBitSet.get(currentRow)) { vector.putNull(vectorOffset); } else { vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow)); } vectorOffset++; } } } }
@Override public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; vector.putDouble(vectorOffset, dataChunk.getDouble(currentRow)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; if (nullBitSet.get(currentRow)) { vector.putNull(vectorOffset); } else { vector.putDouble(vectorOffset, dataChunk.getDouble(currentRow)); } vectorOffset++; } } } }
@Override public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; vector.putInt(vectorOffset, (int)dataChunk.getLong(currentRow)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; if (nullBitSet.get(currentRow)) { vector.putNull(vectorOffset); } else { vector.putInt(vectorOffset, (int)dataChunk.getLong(currentRow)); } vectorOffset++; } } } }
@Override public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; vector.putLong(vectorOffset, dataChunk.getLong(currentRow)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; if (nullBitSet.get(currentRow)) { vector.putNull(vectorOffset); } else { vector.putLong(vectorOffset, dataChunk.getLong(currentRow)); } vectorOffset++; } } } }
@Override public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; BitSet nullBitSet = dataChunk.getNullBits(); if (nullBitSet.isEmpty()) { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; vector.putByte(vectorOffset, dataChunk.getByte(currentRow)); vectorOffset++; } } else { for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; if (nullBitSet.get(currentRow)) { vector.putNull(vectorOffset); } else { vector.putByte(vectorOffset, dataChunk.getByte(currentRow)); } vectorOffset++; } } } }
@Override public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; int precision = info.measure.getMeasure().getPrecision(); int newMeasureScale = info.measure.getMeasure().getScale(); BitSet nullBitSet = dataChunk.getNullBits(); for (int i = offset; i < len; i++) { if (nullBitSet.get(i)) { vector.putNull(vectorOffset); } else { BigDecimal decimal = dataChunk.getDecimal(i); if (decimal.scale() < newMeasureScale) { decimal = decimal.setScale(newMeasureScale); } vector.putDecimal(vectorOffset, decimal, precision); } vectorOffset++; } }
@Override public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk, ColumnVectorInfo info) { int offset = info.offset; int len = offset + info.size; int vectorOffset = info.vectorOffset; CarbonColumnVector vector = info.vector; int precision = info.measure.getMeasure().getPrecision(); BitSet nullBitSet = dataChunk.getNullBits(); for (int i = offset; i < len; i++) { int currentRow = filteredRowId[i]; if (nullBitSet.get(currentRow)) { vector.putNull(vectorOffset); } else { BigDecimal decimal = dataChunk.getDecimal(currentRow); if (info.measure.getMeasure().getScale() > decimal.scale()) { decimal = decimal.setScale(info.measure.getMeasure().getScale()); } vector.putDecimal(vectorOffset, decimal, precision); } vectorOffset++; } } }
private void fillNullBitSet(ColumnPage inputPage, DataChunk2 dataChunk) { PresenceMeta presenceMeta = new PresenceMeta(); presenceMeta.setPresent_bit_streamIsSet(true); Compressor compressor = CompressorFactory.getInstance().getCompressor( inputPage.getColumnCompressorName()); presenceMeta.setPresent_bit_stream( compressor.compressByte(inputPage.getNullBits().toByteArray())); dataChunk.setPresence(presenceMeta); }