public Object getFieldValue(_Fields field) { switch (field) { case MIN_MAX_INDEX: return getMin_max_index(); case B_TREE_INDEX: return getB_tree_index(); } throw new IllegalStateException(); }
public static BlockletIndex getBlockletIndex( org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex info) { BlockletMinMaxIndex blockletMinMaxIndex = convertMinMaxIndex(info.getMinMaxIndex()); BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex(); blockletBTreeIndex.setStart_key(info.getBtreeIndex().getStartKey()); blockletBTreeIndex.setEnd_key(info.getBtreeIndex().getEndKey()); BlockletIndex blockletIndex = new BlockletIndex(); blockletIndex.setMin_max_index(blockletMinMaxIndex); blockletIndex.setB_tree_index(blockletBTreeIndex); return blockletIndex; }
@Override public boolean equals(Object that) { if (that == null) return false; if (that instanceof BlockletIndex) return this.equals((BlockletIndex)that); return false; }
public void setFieldValue(_Fields field, Object value) { switch (field) { case MIN_MAX_INDEX: if (value == null) { unsetMin_max_index(); } else { setMin_max_index((BlockletMinMaxIndex)value); } break; case B_TREE_INDEX: if (value == null) { unsetB_tree_index(); } else { setB_tree_index((BlockletBTreeIndex)value); } break; } }
case 3: // BLOCKLET_INDEX if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { struct.blocklet_index = new BlockletIndex(); struct.blocklet_index.read(iprot); struct.setBlocklet_indexIsSet(true); } else {
index.b_tree_index.getEnd_key()); BlockletMinMaxIndex minMaxIndex = new BlockletMinMaxIndex(index.getMin_max_index().getMin_values(), index.getMin_max_index().getMax_values(), index.getMin_max_index().getMin_max_presence()); org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex bIndex = new org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex(bTreeIndex,
void apppendBlocklet(DataOutputStream outputStream) throws IOException { outputStream.write(CarbonStreamOutputFormat.CARBON_SYNC_MARKER); BlockletInfo blockletInfo = new BlockletInfo(); blockletInfo.setNum_rows(getRowIndex() + 1); BlockletHeader blockletHeader = new BlockletHeader(); blockletHeader.setBlocklet_length(getCount()); blockletHeader.setMutation(MutationType.INSERT); blockletHeader.setBlocklet_info(blockletInfo); // add blocklet level min/max blockletMinMaxIndex = generateBlockletMinMax(); if (blockletInfo.getNum_rows() > 1) { BlockletIndex blockletIndex = new BlockletIndex(); blockletIndex.setMin_max_index(CarbonMetadataUtil.convertMinMaxIndex(blockletMinMaxIndex)); blockletHeader.setBlocklet_index(blockletIndex); } byte[] headerBytes = CarbonUtil.getByteArray(blockletHeader); outputStream.writeInt(headerBytes.length); outputStream.write(headerBytes); byte[] compressed = compressor.compressByte(getBytes(), getCount()); outputStream.writeInt(compressed.length); outputStream.write(compressed); }
public BlockletIndex deepCopy() { return new BlockletIndex(this); }
case 4: // BLOCK_INDEX if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { struct.block_index = new org.apache.carbondata.format.BlockletIndex(); struct.block_index.read(iprot); struct.setBlock_indexIsSet(true); } else {
private boolean isScanRequired(BlockletHeader header) { if (filter != null && header.getBlocklet_index() != null) { BlockletMinMaxIndex minMaxIndex = CarbonMetadataUtil.convertExternalMinMaxIndex( header.getBlocklet_index().getMin_max_index()); if (minMaxIndex != null) { BitSet bitSet = filter .isScanRequired(minMaxIndex.getMaxValues(), minMaxIndex.getMinValues(), minMaxIndex.getIsMinMaxSet()); if (bitSet.isEmpty()) { return false; } else { return true; } } } return true; }
blockIndex.setOffset(-1); BlockletIndex blockletIndex = new BlockletIndex(); blockIndex.setBlock_index(blockletIndex); StreamFileIndex streamFileIndex = indexMap.get(blockIndex.getFile_name()); if (streamFileIndex != null) { blockletIndex.setMin_max_index( CarbonMetadataUtil.convertMinMaxIndex(streamFileIndex.getMinMaxIndex())); blockIndex.setNum_rows(streamFileIndex.getRowCount());
List<BlockletIndex> __this__blocklet_index_list = new ArrayList<BlockletIndex>(other.blocklet_index_list.size()); for (BlockletIndex other_element : other.blocklet_index_list) { __this__blocklet_index_list.add(new BlockletIndex(other_element));
.serializeEndKey(); blockletBTreeIndex.setEnd_key(endKey); BlockletIndex blockletIndex = new BlockletIndex(); blockletIndex.setMin_max_index(blockletMinMaxIndex); blockletIndex.setB_tree_index(blockletBTreeIndex); return blockletIndex;
@Override public void read(org.apache.thrift.protocol.TProtocol prot, BlockIndex struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; struct.num_rows = iprot.readI64(); struct.setNum_rowsIsSet(true); struct.file_name = iprot.readString(); struct.setFile_nameIsSet(true); struct.offset = iprot.readI64(); struct.setOffsetIsSet(true); struct.block_index = new org.apache.carbondata.format.BlockletIndex(); struct.block_index.read(iprot); struct.setBlock_indexIsSet(true); BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { struct.blocklet_info = new org.apache.carbondata.format.BlockletInfo3(); struct.blocklet_info.read(iprot); struct.setBlocklet_infoIsSet(true); } if (incoming.get(1)) { struct.file_size = iprot.readI64(); struct.setFile_sizeIsSet(true); } } }
/** * merge new blocklet index and old file index to create new file index */ private static void updateStreamFileIndex(Map<String, StreamFileIndex> indexMap, String indexPath, FileFactory.FileType fileType, DataType[] msrDataTypes ) throws IOException { List<BlockIndex> blockIndexList = readIndexFile(indexPath, fileType); for (BlockIndex blockIndex : blockIndexList) { BlockletMinMaxIndex fileIndex = CarbonMetadataUtil .convertExternalMinMaxIndex(blockIndex.getBlock_index().getMin_max_index()); StreamFileIndex blockletIndex = indexMap.get(blockIndex.getFile_name()); if (blockletIndex == null) { // should index all stream file indexMap.put(blockIndex.getFile_name(), new StreamFileIndex(blockIndex.getFile_name(), fileIndex, blockIndex.getNum_rows())); } else { // merge minMaxIndex into StreamBlockIndex blockletIndex.setRowCount(blockletIndex.getRowCount() + blockIndex.getNum_rows()); mergeBatchMinMax(blockletIndex, fileIndex, msrDataTypes); } } }
/** * Below method will be used to convert the blocklet index of thrift to * wrapper * * @param blockletIndexThrift * @return blocklet index wrapper */ protected BlockletIndex getBlockletIndex( org.apache.carbondata.format.BlockletIndex blockletIndexThrift) { org.apache.carbondata.format.BlockletBTreeIndex btreeIndex = blockletIndexThrift.getB_tree_index(); org.apache.carbondata.format.BlockletMinMaxIndex minMaxIndex = blockletIndexThrift.getMin_max_index(); List<Boolean> isMinMaxSet = null; // Below logic is added to handle backward compatibility if (minMaxIndex.isSetMin_max_presence()) { isMinMaxSet = minMaxIndex.getMin_max_presence(); } else { Boolean[] minMaxFlag = new Boolean[minMaxIndex.getMax_values().size()]; Arrays.fill(minMaxFlag, true); isMinMaxSet = Arrays.asList(minMaxFlag); } return new BlockletIndex( new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()), new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values(), isMinMaxSet)); }
List<BlockletIndex> __this__blocklet_index_list = new ArrayList<BlockletIndex>(other.blocklet_index_list.size()); for (BlockletIndex other_element : other.blocklet_index_list) { __this__blocklet_index_list.add(new BlockletIndex(other_element));
if (!(this_present_block_index && that_present_block_index)) return false; if (!this.block_index.equals(that.block_index)) return false;
@Override public void read(org.apache.thrift.protocol.TProtocol prot, BlockletHeader struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; struct.blocklet_length = iprot.readI32(); struct.setBlocklet_lengthIsSet(true); struct.mutation = org.apache.carbondata.format.MutationType.findByValue(iprot.readI32()); struct.setMutationIsSet(true); struct.blocklet_info = new BlockletInfo(); struct.blocklet_info.read(iprot); struct.setBlocklet_infoIsSet(true); BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { struct.blocklet_index = new BlockletIndex(); struct.blocklet_index.read(iprot); struct.setBlocklet_indexIsSet(true); } if (incoming.get(1)) { struct.dictionary = new org.apache.carbondata.format.ColumnDictionaryChunk(); struct.dictionary.read(iprot); struct.setDictionaryIsSet(true); } } }
if (withMinMax) { if (blockIndex.getBlock_index() != null && blockIndex.getBlock_index().getMin_max_index() != null) { streamFile.setMinMaxIndex(CarbonMetadataUtil .convertExternalMinMaxIndex(blockIndex.getBlock_index().getMin_max_index()));