private boolean[] getMinMaxFlag(DataMapRow row, int index) { DataMapRow minMaxFlagRow = row.getRow(index); boolean[] minMaxFlag = new boolean[minMaxFlagRow.getColumnCount()]; for (int i = 0; i < minMaxFlag.length; i++) { minMaxFlag[i] = minMaxFlagRow.getBoolean(i); } return minMaxFlag; }
private byte[][] getMinMaxValue(DataMapRow row, int index) { DataMapRow minMaxRow = row.getRow(index); byte[][] minMax = new byte[minMaxRow.getColumnCount()][]; for (int i = 0; i < minMax.length; i++) { minMax[i] = minMaxRow.getByteArray(i); } return minMax; }
public int getSizeInBytes(int ordinal) { switch (schemas[ordinal].getSchemaType()) { case FIXED: return schemas[ordinal].getLength(); case VARIABLE_SHORT: return getLengthInBytes(ordinal) + 2; case VARIABLE_INT: return getLengthInBytes(ordinal) + 4; case STRUCT: DataMapRow row = getRow(ordinal); CarbonRowSchema[] childSchemas = ((CarbonRowSchema.StructCarbonRowSchema) schemas[ordinal]).getChildSchemas(); // set the child schema. Because schema is transient it can be null row.setSchemas(childSchemas); return getRow(ordinal).getTotalSizeInBytes(); default: throw new UnsupportedOperationException("wrong type"); } }
getUnsafe() .putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getByte(index)); runningLength += row.getSizeInBytes(index); } else if (dataType == DataTypes.BOOLEAN) { getUnsafe() .putBoolean(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getBoolean(index)); runningLength += row.getSizeInBytes(index); } else if (dataType == DataTypes.SHORT) { getUnsafe() .putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getShort(index)); runningLength += row.getSizeInBytes(index); } else if (dataType == DataTypes.INT) { getUnsafe() .putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getInt(index)); runningLength += row.getSizeInBytes(index); } else if (dataType == DataTypes.LONG) { getUnsafe() .putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getLong(index)); runningLength += row.getSizeInBytes(index); } else if (dataType == DataTypes.FLOAT) { getUnsafe() .putFloat(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength, row.getFloat(index)); runningLength += row.getSizeInBytes(index);
protected BlockletDetailInfo getBlockletDetailInfo(DataMapRow row, short blockletId, ExtendedBlocklet blocklet) { BlockletDetailInfo detailInfo = new BlockletDetailInfo(); detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX)); detailInfo.setVersionNumber(row.getShort(VERSION_INDEX)); detailInfo.setBlockletId(blockletId); detailInfo.setDimLens(getColumnCardinality()); detailInfo.setSchemaUpdatedTimeStamp(row.getLong(SCHEMA_UPADATED_TIME_INDEX)); try { blocklet.setLocation( new String(row.getByteArray(LOCATIONS), CarbonCommonConstants.DEFAULT_CHARSET) .split(",")); } catch (IOException e) { throw new RuntimeException(e); } detailInfo.setBlockFooterOffset(row.getLong(BLOCK_FOOTER_OFFSET)); detailInfo.setBlockSize(row.getLong(BLOCK_LENGTH)); detailInfo.setLegacyStore(isLegacyStore); return detailInfo; }
@Override public ExtendedBlocklet getDetailedBlocklet(String blockletId) { if (isLegacyStore) { return super.getDetailedBlocklet(blockletId); } int absoluteBlockletId = Integer.parseInt(blockletId); DataMapRow safeRow = memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), absoluteBlockletId) .convertToSafeRow(); short relativeBlockletId = safeRow.getShort(BLOCKLET_ID_INDEX); String filePath = getFilePath(); return createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), relativeBlockletId, false); }
@Override protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId, boolean useMinMaxForPruning) { if (isLegacyStore) { return super.createBlocklet(row, fileName, blockletId, useMinMaxForPruning); } ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + ""); BlockletDetailInfo detailInfo = getBlockletDetailInfo(row, blockletId, blocklet); detailInfo.setColumnSchemas(getColumnSchema()); detailInfo.setBlockletInfoBinary(row.getByteArray(BLOCKLET_INFO_INDEX)); detailInfo.setPagesCount(row.getShort(BLOCKLET_PAGE_COUNT_INDEX)); detailInfo.setUseMinMaxForPruning(useMinMaxForPruning); blocklet.setDetailInfo(detailInfo); return blocklet; }
private String[] getFileDetails() { try { String[] fileDetails = new String[3]; DataMapRow unsafeRow = taskSummaryDMStore.getDataMapRow(getTaskSummarySchema(), 0); fileDetails[0] = new String(unsafeRow.getByteArray(SUMMARY_INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET); fileDetails[1] = new String(unsafeRow.getByteArray(SUMMARY_INDEX_FILE_NAME), CarbonCommonConstants.DEFAULT_CHARSET); fileDetails[2] = new String(unsafeRow.getByteArray(SUMMARY_SEGMENTID), CarbonCommonConstants.DEFAULT_CHARSET); return fileDetails; } catch (Exception e) { throw new RuntimeException(e); } }
memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), rowIndex).convertToSafeRow(); String filePath = getFilePath(); return createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), relativeBlockletId,
protected String getFileNameWithFilePath(DataMapRow dataMapRow, String filePath) { String fileName = filePath + CarbonCommonConstants.FILE_SEPARATOR + new String( dataMapRow.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS) + CarbonTablePath.getCarbonDataExtension(); return fileName; }
if (filterExp == null) { for (int i = 0; i < numEntries; i++) { DataMapRow safeRow = memoryDMStore.getDataMapRow(schema, i).convertToSafeRow(); blocklets.add(createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), getBlockletId(safeRow), false)); DataMapRow safeRow = memoryDMStore.getDataMapRow(schema, entryIndex).convertToSafeRow(); boolean[] minMaxFlag = getMinMaxFlag(safeRow, BLOCK_MIN_MAX_FLAG); String fileName = getFileNameWithFilePath(safeRow, filePath);
/** * Get the index file name of the blocklet data map * * @return */ public String getTableTaskInfo(int index) { DataMapRow unsafeRow = taskSummaryDMStore.getDataMapRow(getTaskSummarySchema(), 0); try { return new String(unsafeRow.getByteArray(index), CarbonCommonConstants.DEFAULT_CHARSET); } catch (UnsupportedEncodingException e) { // should never happen! throw new IllegalArgumentException("UTF8 encoding is not supported", e); } }
private byte[] getBlockletRowCountForEachBlock() { // taskSummary DM store will have only one row CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema(); return taskSummaryDMStore .getDataMapRow(taskSummarySchema, taskSummaryDMStore.getRowCount() - 1) .getByteArray(taskSummarySchema.length - 1); }