/** * This method will ocnvert safe to unsafe memory DM store * * @throws MemoryException */ public void convertToUnsafeDMStore() throws MemoryException { if (memoryDMStore instanceof SafeMemoryDMStore) { UnsafeMemoryDMStore unsafeMemoryDMStore = memoryDMStore.convertToUnsafeDMStore( getFileFooterEntrySchema()); memoryDMStore.freeMemory(); memoryDMStore = unsafeMemoryDMStore; } if (taskSummaryDMStore instanceof SafeMemoryDMStore) { UnsafeMemoryDMStore unsafeSummaryMemoryDMStore = taskSummaryDMStore.convertToUnsafeDMStore(getTaskSummarySchema()); taskSummaryDMStore.freeMemory(); taskSummaryDMStore = unsafeSummaryMemoryDMStore; } }
private void finishWriting(CarbonRowSchema[] taskSummarySchema, byte[] filePath, byte[] fileName, byte[] segmentId, DataMapRowImpl summaryRow) throws MemoryException { if (memoryDMStore != null) { memoryDMStore.finishWriting(); } if (null != taskSummaryDMStore) { addTaskSummaryRowToUnsafeMemoryStore(taskSummarySchema, summaryRow, filePath, fileName, segmentId); taskSummaryDMStore.finishWriting(); } }
private byte[] getBlockletRowCountForEachBlock() { // taskSummary DM store will have only one row CarbonRowSchema[] taskSummarySchema = getTaskSummarySchema(); return taskSummaryDMStore .getDataMapRow(taskSummarySchema, taskSummaryDMStore.getRowCount() - 1) .getByteArray(taskSummarySchema.length - 1); }
/** * Get the index file name of the blocklet data map * * @return */ public String getTableTaskInfo(int index) { DataMapRow unsafeRow = taskSummaryDMStore.getDataMapRow(getTaskSummarySchema(), 0); try { return new String(unsafeRow.getByteArray(index), CarbonCommonConstants.DEFAULT_CHARSET); } catch (UnsupportedEncodingException e) { // should never happen! throw new IllegalArgumentException("UTF8 encoding is not supported", e); } }
public int getTotalBlocks() { if (isLegacyStore) { // dummy value return 0; } else { return memoryDMStore.getRowCount(); } }
private void addTaskSummaryRowToUnsafeMemoryStore(CarbonRowSchema[] taskSummarySchema, DataMapRow summaryRow, byte[] filePath, byte[] fileName, byte[] segmentId) { // write the task summary info to unsafe memory store if (null != summaryRow) { summaryRow.setByteArray(fileName, SUMMARY_INDEX_FILE_NAME); summaryRow.setByteArray(segmentId, SUMMARY_SEGMENTID); if (null != filePath) { summaryRow.setByteArray(filePath, SUMMARY_INDEX_PATH); } try { taskSummaryDMStore.addIndexRow(taskSummarySchema, summaryRow); } catch (Exception e) { throw new RuntimeException(e); } } }
public long getMemorySize() { long memoryUsed = 0L; if (memoryDMStore != null) { memoryUsed += memoryDMStore.getMemoryUsed(); } if (null != taskSummaryDMStore) { memoryUsed += taskSummaryDMStore.getMemoryUsed(); } return memoryUsed; }
@Override public void clear() { if (memoryDMStore != null) { memoryDMStore.freeMemory(); } // clear task min/max unsafe memory if (null != taskSummaryDMStore) { taskSummaryDMStore.freeMemory(); } }
private String[] getFileDetails() { try { String[] fileDetails = new String[3]; DataMapRow unsafeRow = taskSummaryDMStore.getDataMapRow(getTaskSummarySchema(), 0); fileDetails[0] = new String(unsafeRow.getByteArray(SUMMARY_INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET); fileDetails[1] = new String(unsafeRow.getByteArray(SUMMARY_INDEX_FILE_NAME), CarbonCommonConstants.DEFAULT_CHARSET); fileDetails[2] = new String(unsafeRow.getByteArray(SUMMARY_SEGMENTID), CarbonCommonConstants.DEFAULT_CHARSET); return fileDetails; } catch (Exception e) { throw new RuntimeException(e); } }
@Override protected int getTotalBlocklets() { if (isLegacyStore) { return super.getTotalBlocklets(); } else { return memoryDMStore.getRowCount(); } }
memoryDMStore.addIndexRow(schema, row); } catch (Exception e) { throw new RuntimeException(e);
@Override public boolean isScanRequired(FilterResolverIntf filterExp) { FilterExecuter filterExecuter = FilterUtil .getFilterExecuterTree(filterExp, getSegmentProperties(), null, getMinMaxCacheColumns()); DataMapRow unsafeRow = taskSummaryDMStore .getDataMapRow(getTaskSummarySchema(), taskSummaryDMStore.getRowCount() - 1); boolean isScanRequired = FilterExpressionProcessor .isScanRequired(filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX), getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX), getMinMaxFlag(unsafeRow, TASK_MIN_MAX_FLAG)); if (isScanRequired) { return true; } return false; }
memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), rowIndex).convertToSafeRow(); String filePath = getFilePath(); return createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), relativeBlockletId,
@Override public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties, List<PartitionSpec> partitions) { if (memoryDMStore.getRowCount() == 0) { return new ArrayList<>(); } // if it has partitioned datamap but there is no partitioned information stored, it means // partitions are dropped so return empty list. if (partitions != null) { if (!validatePartitionInfo(partitions)) { return new ArrayList<>(); } } // Prune with filters if the partitions are existed in this datamap // changed segmentProperties to this.segmentProperties to make sure the pruning with its own // segmentProperties. // Its a temporary fix. The Interface DataMap.prune(FilterResolverIntf filterExp, // SegmentProperties segmentProperties, List<PartitionSpec> partitions) should be corrected return prune(filterExp); }
memoryDMStore.addIndexRow(schema, row); } catch (Exception e) { throw new RuntimeException(e);
private List<Blocklet> prune(FilterResolverIntf filterExp) { if (memoryDMStore.getRowCount() == 0) { return new ArrayList<>(); CarbonRowSchema[] schema = getFileFooterEntrySchema(); String filePath = getFilePath(); int numEntries = memoryDMStore.getRowCount(); int totalBlocklets = getTotalBlocklets(); int hitBlocklets = 0; if (filterExp == null) { for (int i = 0; i < numEntries; i++) { DataMapRow safeRow = memoryDMStore.getDataMapRow(schema, i).convertToSafeRow(); blocklets.add(createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), getBlockletId(safeRow), false)); DataMapRow safeRow = memoryDMStore.getDataMapRow(schema, entryIndex).convertToSafeRow(); boolean[] minMaxFlag = getMinMaxFlag(safeRow, BLOCK_MIN_MAX_FLAG); String fileName = getFileNameWithFilePath(safeRow, filePath);
@Override public ExtendedBlocklet getDetailedBlocklet(String blockletId) { if (isLegacyStore) { return super.getDetailedBlocklet(blockletId); } int absoluteBlockletId = Integer.parseInt(blockletId); DataMapRow safeRow = memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), absoluteBlockletId) .convertToSafeRow(); short relativeBlockletId = safeRow.getShort(BLOCKLET_ID_INDEX); String filePath = getFilePath(); return createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), relativeBlockletId, false); }