protected AbstractMemoryDMStore getMemoryDMStore(boolean addToUnsafe) throws MemoryException { AbstractMemoryDMStore memoryDMStore; if (addToUnsafe) { memoryDMStore = new UnsafeMemoryDMStore(); } else { memoryDMStore = new SafeMemoryDMStore(); } return memoryDMStore; }
public int getTotalBlocks() { if (isLegacyStore) { // dummy value return 0; } else { return memoryDMStore.getRowCount(); } }
@Override public void clear() { if (memoryDMStore != null) { memoryDMStore.freeMemory(); } // clear task min/max unsafe memory if (null != taskSummaryDMStore) { taskSummaryDMStore.freeMemory(); } }
private CarbonInputSplit convertToCarbonInputSplit(ExtendedBlocklet blocklet) throws IOException { CarbonInputSplit split = CarbonInputSplit .from(blocklet.getSegmentId(), blocklet.getBlockletId(), new FileSplit(new Path(blocklet.getPath()), 0, blocklet.getLength(), blocklet.getLocations()), ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()), blocklet.getDataMapWriterPath()); split.setDetailInfo(blocklet.getDetailInfo()); return split; }
@Override public UnsafeMemoryDMStore convertToUnsafeDMStore(CarbonRowSchema[] schema) throws MemoryException { setSchemaDataType(schema); UnsafeMemoryDMStore unsafeMemoryDMStore = new UnsafeMemoryDMStore(); for (DataMapRow dataMapRow : dataMapRows) { dataMapRow.setSchemas(schema); unsafeMemoryDMStore.addIndexRow(schema, dataMapRow); } unsafeMemoryDMStore.finishWriting(); return unsafeMemoryDMStore; }
protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId, boolean useMinMaxForPruning) { ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + "", false); BlockletDetailInfo detailInfo = getBlockletDetailInfo(row, blockletId, blocklet); detailInfo.setBlockletInfoBinary(new byte[0]); blocklet.setDetailInfo(detailInfo); return blocklet; }
public String getPath() { return getFilePath(); }
/** * The method clears the access count of table segments * * @param tableSegmentUniqueIdentifiersWrapper */ @Override public void clearAccessCount( List<TableBlockIndexUniqueIdentifierWrapper> tableSegmentUniqueIdentifiersWrapper) { for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : tableSegmentUniqueIdentifiersWrapper) { BlockDataMap cacheable = (BlockDataMap) lruCache.get( identifierWrapper.getTableBlockIndexUniqueIdentifier().getUniqueTableSegmentIdentifier()); cacheable.clear(); } } }
private void finishWriting(CarbonRowSchema[] taskSummarySchema, byte[] filePath, byte[] fileName, byte[] segmentId, DataMapRowImpl summaryRow) throws MemoryException { if (memoryDMStore != null) { memoryDMStore.finishWriting(); } if (null != taskSummaryDMStore) { addTaskSummaryRowToUnsafeMemoryStore(taskSummarySchema, summaryRow, filePath, fileName, segmentId); taskSummaryDMStore.finishWriting(); } }
@Override protected int getTotalBlocklets() { if (isLegacyStore) { return super.getTotalBlocklets(); } else { return memoryDMStore.getRowCount(); } }
/** * returns the SegmentTaskIndexWrapper * * @param tableSegmentUniqueIdentifierWrapper * @return */ @Override public BlockletDataMapIndexWrapper getIfPresent( TableBlockIndexUniqueIdentifierWrapper tableSegmentUniqueIdentifierWrapper) { return (BlockletDataMapIndexWrapper) lruCache.get( tableSegmentUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() .getUniqueTableSegmentIdentifier()); }
private static boolean pathExistsInPartitionSpec(List<PartitionSpec> partitionSpecs, Path partitionPath) { for (PartitionSpec spec : partitionSpecs) { if (spec.getLocation().equals(partitionPath)) { return true; } } return false; }
@Override public int hashCode() { return super.hashCode(); } }
@Override public BlockletDataMapIndexWrapper get(TableBlockIndexUniqueIdentifierWrapper identifierWrapper) throws IOException { return get(identifierWrapper, null); }
public List<ColumnSchema> getColumnSchemas() throws IOException { if (columnSchemas == null && columnSchemaBinary != null) { readColumnSchema(columnSchemaBinary); } return columnSchemas; }
public long getMemorySize() { long memoryUsed = 0L; if (memoryDMStore != null) { memoryUsed += memoryDMStore.getMemoryUsed(); } if (null != taskSummaryDMStore) { memoryUsed += taskSummaryDMStore.getMemoryUsed(); } return memoryUsed; }
private List<ExtendedBlocklet> addSegmentId(List<ExtendedBlocklet> pruneBlocklets, String segmentId) { for (ExtendedBlocklet blocklet : pruneBlocklets) { blocklet.setSegmentId(segmentId); } return pruneBlocklets; } /**
@Override public boolean equals(Object o) { return super.equals(o); }
public BlockletInfo getBlockletInfo() { if (null == blockletInfo) { try { setBlockletInfoFromBinary(); } catch (IOException e) { throw new RuntimeException(e); } } return blockletInfo; }
@Override public int hashCode() { int result = super.hashCode(); result = 31 * result + (segmentId != null ? segmentId.hashCode() : 0); return result; } }