public DataTableBuilder(@Nonnull DataSchema dataSchema) { _dataSchema = dataSchema; _columnOffsets = new int[dataSchema.size()]; _rowSizeInBytes = DataTableUtils.computeColumnOffsets(dataSchema, _columnOffsets); }
/** * Construct data table with results. (Server side) */ public DataTableImplV2(int numRows, @Nonnull DataSchema dataSchema, @Nonnull Map<String, Map<Integer, String>> dictionaryMap, @Nonnull byte[] fixedSizeDataBytes, @Nonnull byte[] variableSizeDataBytes) { _numRows = numRows; _numColumns = dataSchema.size(); _dataSchema = dataSchema; _columnOffsets = new int[_numColumns]; _rowSizeInBytes = DataTableUtils.computeColumnOffsets(dataSchema, _columnOffsets); _dictionaryMap = dictionaryMap; _fixedSizeDataBytes = fixedSizeDataBytes; _fixedSizeData = ByteBuffer.wrap(fixedSizeDataBytes); _variableSizeDataBytes = variableSizeDataBytes; _variableSizeData = ByteBuffer.wrap(variableSizeDataBytes); _metadata = new HashMap<>(); }
/** * Helper method to compute column indices from selection columns and the data schema for selection queries with * <code>ORDER BY</code>. * @param selectionColumns selection columns. * @param dataSchema data schema. * @return column indices */ public static int[] getColumnIndicesWithOrdering(@Nonnull List<String> selectionColumns, @Nonnull DataSchema dataSchema) { int numSelectionColumns = selectionColumns.size(); int[] columnIndices = new int[numSelectionColumns]; int numColumnsInDataSchema = dataSchema.size(); Map<String, Integer> dataSchemaIndices = new HashMap<>(numColumnsInDataSchema); for (int i = 0; i < numColumnsInDataSchema; i++) { dataSchemaIndices.put(dataSchema.getColumnName(i), i); } for (int i = 0; i < numSelectionColumns; i++) { columnIndices[i] = dataSchemaIndices.get(selectionColumns.get(i)); } return columnIndices; }
/** * Expand <code>'SELECT *'</code> to select all columns with {@link DataSchema}, order all columns alphabatically. * (Inter segment) * * @param selectionColumns unexpanded selection columns (may contain '*'). * @param dataSchema data schema. * @return expanded selection columns. */ @Nonnull public static List<String> getSelectionColumns(@Nonnull List<String> selectionColumns, @Nonnull DataSchema dataSchema) { if ((selectionColumns.size() == 1) && selectionColumns.get(0).equals("*")) { int numColumns = dataSchema.size(); List<String> allColumns = new ArrayList<>(numColumns); for (int i = 0; i < numColumns; i++) { allColumns.add(dataSchema.getColumnName(i)); } Collections.sort(allColumns); return allColumns; } else { return selectionColumns; } }
assert numColumns == dataSchema.size();
private int getVirtualColumns(DataSchema selectionDataSchema) { int virtualCols = 0; for (int i = 0; i < selectionDataSchema.size(); ++i) { if (selectionDataSchema.getColumnName(i).startsWith("$")) { virtualCols++; } } return virtualCols; } }
byteBuffer.get(schemaBytes); _dataSchema = DataSchema.fromBytes(schemaBytes); _columnOffsets = new int[_dataSchema.size()]; _rowSizeInBytes = DataTableUtils.computeColumnOffsets(_dataSchema, _columnOffsets); } else {
@Test public void testGetters() { DataSchema dataSchema = new DataSchema(COLUMN_NAMES, COLUMN_DATA_TYPES); Assert.assertEquals(dataSchema.size(), NUM_COLUMNS); for (int i = 0; i < NUM_COLUMNS; i++) { Assert.assertEquals(dataSchema.getColumnName(i), COLUMN_NAMES[i]); Assert.assertEquals(dataSchema.getColumnDataType(i), COLUMN_DATA_TYPES[i]); } }
@Override protected IntermediateResultsBlock getNextBlock() { int numDocsScanned = 0; ProjectionBlock projectionBlock; while ((projectionBlock = _projectionOperator.nextBlock()) != null) { for (int i = 0; i < _dataSchema.size(); i++) { _blocks[i] = projectionBlock.getBlock(_dataSchema.getColumnName(i)); } SelectionFetcher selectionFetcher = new SelectionFetcher(_blocks, _dataSchema); DocIdSetBlock docIdSetBlock = projectionBlock.getDocIdSetBlock(); int numDocsToFetch = Math.min(docIdSetBlock.getSearchableLength(), _limitDocs - _rowEvents.size()); numDocsScanned += numDocsToFetch; int[] docIdSet = docIdSetBlock.getDocIdSet(); for (int i = 0; i < numDocsToFetch; i++) { _rowEvents.add(selectionFetcher.getRow(docIdSet[i])); } if (_rowEvents.size() == _limitDocs) { break; } } // Create execution statistics. long numEntriesScannedInFilter = _projectionOperator.getExecutionStatistics().getNumEntriesScannedInFilter(); long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumColumnsProjected(); long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs(); _executionStatistics = new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs); return new IntermediateResultsBlock(_dataSchema, _rowEvents); }
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 4); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 4); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 4); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column5"); Assert.assertEquals(selectionDataSchema.getColumnName(3), "column6"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 4); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column5"); Assert.assertEquals(selectionDataSchema.getColumnName(3), "column6");
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 3); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(2), "column6"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 3); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(2), "column6");
DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(getVirtualColumns(selectionDataSchema), 0); Assert.assertEquals(selectionDataSchema.size(), 11); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1"); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(getVirtualColumns(selectionDataSchema), 0); Assert.assertEquals(selectionDataSchema.size(), 11); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 3); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(2), "column11"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 3); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(2), "column11");
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 10); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(5), "column6"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 10); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(5), "column6");
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 11); Assert.assertEquals(getVirtualColumns(selectionDataSchema), 0); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 11); Assert.assertEquals(getVirtualColumns(selectionDataSchema), 0); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 11); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column11"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 11); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column11");
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 10); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(5), "column6"); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 10); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1"); Assert.assertEquals(selectionDataSchema.getColumnName(5), "column6");
public static Serializable[] extractRowFromDataTable(@Nonnull DataTable dataTable, int rowId) { DataSchema dataSchema = dataTable.getDataSchema(); int numColumns = dataSchema.size();
@Override protected IntermediateResultsBlock getNextBlock() { int numDocsScanned = 0; ProjectionBlock projectionBlock; while ((projectionBlock = _projectionOperator.nextBlock()) != null) { for (int i = 0; i < _dataSchema.size(); i++) { _blocks[i] = projectionBlock.getBlock(_dataSchema.getColumnName(i)); } DocIdSetBlock docIdSetBlock = projectionBlock.getDocIdSetBlock(); _selectionOperatorService.iterateOnBlocksWithOrdering(docIdSetBlock.getBlockDocIdSet().iterator(), _blocks); } // Create execution statistics. numDocsScanned += _selectionOperatorService.getNumDocsScanned(); long numEntriesScannedInFilter = _projectionOperator.getExecutionStatistics().getNumEntriesScannedInFilter(); long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumColumnsProjected(); long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs(); _executionStatistics = new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs); return new IntermediateResultsBlock(_selectionOperatorService.getDataSchema(), _selectionOperatorService.getRows()); }