private boolean[] getMinMaxFlag(DataMapRow row, int index) { DataMapRow minMaxFlagRow = row.getRow(index); boolean[] minMaxFlag = new boolean[minMaxFlagRow.getColumnCount()]; for (int i = 0; i < minMaxFlag.length; i++) { minMaxFlag[i] = minMaxFlagRow.getBoolean(i); } return minMaxFlag; }
@Override public int getTotalSizeInBytes() { if (totalLengthInBytes > 0) { return totalLengthInBytes; } else { return super.getTotalSizeInBytes(); } } }
private int getPosition(int ordinal) { int position = 0; for (int i = 0; i < ordinal; i++) { position += getSizeInBytes(i, position); } return position; } }
private byte[][] getMinMaxValue(DataMapRow row, int index) { DataMapRow minMaxRow = row.getRow(index); byte[][] minMax = new byte[minMaxRow.getColumnCount()][]; for (int i = 0; i < minMax.length; i++) { minMax[i] = minMaxRow.getByteArray(i); } return minMax; }
protected void addMinMaxFlagValues(DataMapRow row, CarbonRowSchema carbonRowSchema, boolean[] minMaxFlag, int ordinal) { CarbonRowSchema[] minMaxFlagSchema = ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas(); DataMapRow minMaxFlagRow = new DataMapRowImpl(minMaxFlagSchema); int flagOrdinal = 0; // min value adding for (int i = 0; i < minMaxFlag.length; i++) { minMaxFlagRow.setBoolean(minMaxFlag[i], flagOrdinal++); } row.setRow(minMaxFlagRow, ordinal); }
private int getSizeInBytes(int ordinal, int position) { switch (schemas[ordinal].getSchemaType()) { case FIXED: return schemas[ordinal].getLength(); case VARIABLE_SHORT: return getLengthInBytes(ordinal, position) + 2; case VARIABLE_INT: return getLengthInBytes(ordinal, position) + 4; case STRUCT: return getRow(ordinal).getTotalSizeInBytes(); default: throw new UnsupportedOperationException("wrong type"); } }
protected DataMapRow addMinMax(CarbonRowSchema carbonRowSchema, byte[][] minValues) { CarbonRowSchema[] minSchemas = ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas(); DataMapRow minRow = new DataMapRowImpl(minSchemas); int minOrdinal = 0; // min value adding for (int i = 0; i < minValues.length; i++) { minRow.setByteArray(minValues[i], minOrdinal++); } return minRow; }
@Override public DataMapRow getRow(int ordinal) { CarbonRowSchema[] childSchemas = ((CarbonRowSchema.StructCarbonRowSchema) schemas[ordinal]).getChildSchemas(); return new UnsafeDataMapRow(childSchemas, block, pointer + getPosition(ordinal)); }
protected String getFileNameWithFilePath(DataMapRow dataMapRow, String filePath) { String fileName = filePath + CarbonCommonConstants.FILE_SEPARATOR + new String( dataMapRow.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS) + CarbonTablePath.getCarbonDataExtension(); return fileName; }
protected void setLocations(String[] locations, DataMapRow row, int ordinal) throws UnsupportedEncodingException { // Add location info String locationStr = StringUtils.join(locations, ','); row.setByteArray(locationStr.getBytes(CarbonCommonConstants.DEFAULT_CHARSET), ordinal); }
@Override public long getLong(int ordinal) { return getUnsafe() .getLong(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); }
public int getTotalSizeInBytes() { int len = 0; for (int i = 0; i < schemas.length; i++) { len += getSizeInBytes(i); } return len; }
public DataMapRow getDataMapRow(CarbonRowSchema[] schema, int index) { assert (index < rowCount); return new UnsafeDataMapRow(schema, memoryBlock, pointers[index]); }
@Override protected short getBlockletId(DataMapRow dataMapRow) { if (isLegacyStore) { return super.getBlockletId(dataMapRow); } return dataMapRow.getShort(BLOCKLET_ID_INDEX); }
@Override public UnsafeMemoryDMStore convertToUnsafeDMStore(CarbonRowSchema[] schema) throws MemoryException { setSchemaDataType(schema); UnsafeMemoryDMStore unsafeMemoryDMStore = new UnsafeMemoryDMStore(); for (DataMapRow dataMapRow : dataMapRows) { dataMapRow.setSchemas(schema); unsafeMemoryDMStore.addIndexRow(schema, dataMapRow); } unsafeMemoryDMStore.finishWriting(); return unsafeMemoryDMStore; }
/** * Add the index row to dataMapRows, basically to in memory. * * @param indexRow * @return */ @Override public void addIndexRow(CarbonRowSchema[] schema, DataMapRow indexRow) throws MemoryException { dataMapRows.add(indexRow); runningLength += indexRow.getTotalSizeInBytes(); }
private byte[] getBlockletRowCountForEachBlock() { // taskSummary DM store will have only one row CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema(); return taskSummaryDMStore .getDataMapRow(taskSummarySchema, taskSummaryDMStore.getRowCount() - 1) .getByteArray(taskSummarySchema.length - 1); }
private void addTaskSummaryRowToUnsafeMemoryStore(CarbonRowSchema[] taskSummarySchema, DataMapRow summaryRow, byte[] filePath, byte[] fileName, byte[] segmentId) { // write the task summary info to unsafe memory store if (null != summaryRow) { summaryRow.setByteArray(fileName, SUMMARY_INDEX_FILE_NAME); summaryRow.setByteArray(segmentId, SUMMARY_SEGMENTID); if (null != filePath) { summaryRow.setByteArray(filePath, SUMMARY_INDEX_PATH); } try { taskSummaryDMStore.addIndexRow(taskSummarySchema, summaryRow); } catch (Exception e) { throw new RuntimeException(e); } } }
@Override public double getDouble(int ordinal) { return getUnsafe() .getDouble(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal)); }
/** * Add the index row to unsafe. * * @param indexRow * @return */ public void addIndexRow(CarbonRowSchema[] schema, DataMapRow indexRow) throws MemoryException { // First calculate the required memory to keep the row in unsafe int rowSize = indexRow.getTotalSizeInBytes(); // Check whether allocated memory is sufficient or not. ensureSize(rowSize); int pointer = runningLength; for (int i = 0; i < schema.length; i++) { addToUnsafe(schema[i], indexRow, i); } pointers[rowCount++] = pointer; }