@Test public void testGetters() { DataSchema dataSchema = new DataSchema(COLUMN_NAMES, COLUMN_DATA_TYPES); Assert.assertEquals(dataSchema.size(), NUM_COLUMNS); for (int i = 0; i < NUM_COLUMNS; i++) { Assert.assertEquals(dataSchema.getColumnName(i), COLUMN_NAMES[i]); Assert.assertEquals(dataSchema.getColumnDataType(i), COLUMN_DATA_TYPES[i]); } }
@Test public void testClone() { DataSchema dataSchema = new DataSchema(COLUMN_NAMES, COLUMN_DATA_TYPES); DataSchema dataSchemaClone = dataSchema.clone(); Assert.assertEquals(dataSchema, dataSchemaClone); Assert.assertEquals(dataSchema.hashCode(), dataSchemaClone.hashCode()); }
@Test public void testTypeCompatible() { DataSchema dataSchema = new DataSchema(COLUMN_NAMES, COLUMN_DATA_TYPES); DataSchema compatibleDataSchema = new DataSchema(COLUMN_NAMES, COMPATIBLE_COLUMN_DATA_TYPES); Assert.assertTrue(dataSchema.isTypeCompatibleWith(compatibleDataSchema)); String[] anotherColumnNames = new String[NUM_COLUMNS]; Arrays.fill(anotherColumnNames, "foo"); DataSchema incompatibleDataSchema = new DataSchema(anotherColumnNames, COLUMN_DATA_TYPES); Assert.assertFalse(dataSchema.isTypeCompatibleWith(incompatibleDataSchema)); dataSchema.upgradeToCover(compatibleDataSchema); DataSchema upgradedDataSchema = new DataSchema(COLUMN_NAMES, UPGRADED_COLUMN_DATA_TYPES); Assert.assertEquals(dataSchema, upgradedDataSchema); }
/** * Helper method to compute column indices from selection columns and the data schema for selection queries with * <code>ORDER BY</code>. * @param selectionColumns selection columns. * @param dataSchema data schema. * @return column indices */ public static int[] getColumnIndicesWithOrdering(@Nonnull List<String> selectionColumns, @Nonnull DataSchema dataSchema) { int numSelectionColumns = selectionColumns.size(); int[] columnIndices = new int[numSelectionColumns]; int numColumnsInDataSchema = dataSchema.size(); Map<String, Integer> dataSchemaIndices = new HashMap<>(numColumnsInDataSchema); for (int i = 0; i < numColumnsInDataSchema; i++) { dataSchemaIndices.put(dataSchema.getColumnName(i), i); } for (int i = 0; i < numSelectionColumns; i++) { columnIndices[i] = dataSchemaIndices.get(selectionColumns.get(i)); } return columnIndices; }
assert numColumns == dataSchema.size(); switch (dataSchema.getColumnDataType(i)) { case INT: rowSizeInBytes += 4;
@Test public void testSerDe() throws Exception { DataSchema dataSchema = new DataSchema(COLUMN_NAMES, COLUMN_DATA_TYPES); DataSchema dataSchemaAfterSerDe = DataSchema.fromBytes(dataSchema.toBytes()); Assert.assertEquals(dataSchema, dataSchemaAfterSerDe); Assert.assertEquals(dataSchema.hashCode(), dataSchemaAfterSerDe.hashCode()); }
public static Serializable[] extractRowFromDataTable(@Nonnull DataTable dataTable, int rowId) { DataSchema dataSchema = dataTable.getDataSchema(); int numColumns = dataSchema.size(); DataSchema.ColumnDataType columnDataType = dataSchema.getColumnDataType(i); switch (columnDataType) { "Unsupported column data type: " + columnDataType + " for column: " + dataSchema.getColumnName(i));
@SuppressWarnings("CloneDoesntCallSuperClone") @Override public DataSchema clone() { return new DataSchema(_columnNames.clone(), _columnDataTypes.clone()); }
@Test public void testCompatibleRowsDataTableTransformation() throws Exception { Collection<Serializable[]> rows = new ArrayList<>(2); rows.add(_row1.clone()); rows.add(_compatibleRow1.clone()); DataSchema dataSchema = _dataSchema.clone(); Assert.assertTrue(dataSchema.isTypeCompatibleWith(_compatibleDataSchema)); dataSchema.upgradeToCover(_compatibleDataSchema); Assert.assertEquals(dataSchema, _upgradedDataSchema); DataTable dataTable = SelectionOperatorUtils.getDataTableFromRows(rows, dataSchema); Serializable[] expectedRow1 = {0L, 1.0, 2.0, 3.0, "4", new long[]{5L}, new double[]{6.0}, new double[]{7.0}, new double[]{8.0}, new String[]{"9"}}; Serializable[] expectedCompatibleRow1 = {1L, 2.0, 3.0, 4.0, "5", new long[]{6L}, new double[]{7.0}, new double[]{8.0}, new double[]{9.0}, new String[]{"10"}}; Assert.assertEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 0), expectedRow1); Assert.assertEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 1), expectedCompatibleRow1); }
@Test public void testToString() { DataSchema dataSchema = new DataSchema(COLUMN_NAMES, COLUMN_DATA_TYPES); Assert.assertEquals(dataSchema.toString(), "[int(INT),long(LONG),float(FLOAT),double(DOUBLE),string(STRING),object(OBJECT),int_array(INT_ARRAY),long_array(LONG_ARRAY),float_array(FLOAT_ARRAY),double_array(DOUBLE_ARRAY),string_array(STRING_ARRAY)]"); }
/** * Given a data schema, remove data tables that are not compatible with this data schema. * <p>Upgrade the data schema passed in to cover all remaining data schemas. * * @param dataSchema data schema. * @param dataTableMap map from server to data table. * @return list of server names where the data table got removed. */ @Nonnull private List<String> removeConflictingResponses(@Nonnull DataSchema dataSchema, @Nonnull Map<ServerInstance, DataTable> dataTableMap) { List<String> droppedServers = new ArrayList<>(); Iterator<Map.Entry<ServerInstance, DataTable>> iterator = dataTableMap.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<ServerInstance, DataTable> entry = iterator.next(); DataSchema dataSchemaToCompare = entry.getValue().getDataSchema(); assert dataSchemaToCompare != null; if (!dataSchema.isTypeCompatibleWith(dataSchemaToCompare)) { droppedServers.add(entry.getKey().toString()); iterator.remove(); } else { dataSchema.upgradeToCover(dataSchemaToCompare); } } return droppedServers; }
for (int i = 0; i < numAggregationFunctions; i++) { Object intermediateResultToMerge; DataSchema.ColumnDataType columnDataType = dataSchema.getColumnDataType(i); switch (columnDataType) { case LONG: resultValue = AggregationFunctionUtils.formatValue(resultValue); reducedAggregationResults.add(new AggregationResult(dataSchema.getColumnName(i), resultValue));
@Nonnull @Override public String getString(int rowId, int colId) { _fixedSizeData.position(rowId * _rowSizeInBytes + _columnOffsets[colId]); int dictId = _fixedSizeData.getInt(); return _dictionaryMap.get(_dataSchema.getColumnName(colId)).get(dictId); }
stringBuilder.append(_dataSchema.toString()).append('\n'); stringBuilder.append("numRows: ").append(_numRows).append('\n'); for (int rowId = 0; rowId < _numRows; rowId++) { for (int colId = 0; colId < _numColumns; colId++) { switch (_dataSchema.getColumnDataType(colId)) { case INT: stringBuilder.append(_fixedSizeData.getInt());
/** * Helper method to format a selection row, make all values string or string array type based on data schema passed in * for selection queries without <code>ORDER BY</code>. (Broker side) * <p>Formatted row is used to build the {@link SelectionResults}. * * @param row selection row to be formatted. * @param dataSchema data schema. */ private static Serializable[] formatRowWithoutOrdering(@Nonnull Serializable[] row, @Nonnull DataSchema dataSchema) { int numColumns = row.length; for (int i = 0; i < numColumns; i++) { row[i] = getFormattedValue(row[i], dataSchema.getColumnDataType(i)); } return row; }
public DataTableBuilder(@Nonnull DataSchema dataSchema) { _dataSchema = dataSchema; _columnOffsets = new int[dataSchema.size()]; _rowSizeInBytes = DataTableUtils.computeColumnOffsets(dataSchema, _columnOffsets); }
DataSchema masterDataSchema = cachedDataSchema.clone(); if (dataTableMap.size() > 1) { List<String> droppedServers = removeConflictingResponses(masterDataSchema, dataTableMap);
byte[] dataSchemaBytes = null; if (_dataSchema != null) { dataSchemaBytes = _dataSchema.toBytes(); dataOutputStream.writeInt(dataSchemaBytes.length); dataOffset += dataSchemaBytes.length;
Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 4); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1"); Assert.assertEquals(selectionDataSchema.getColumnDataType(0), DataSchema.ColumnDataType.INT); Assert.assertEquals(selectionDataSchema.getColumnDataType(1), DataSchema.ColumnDataType.INT); Queue<Serializable[]> selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult(); Assert.assertEquals(selectionResult.size(), 10); Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L); selectionDataSchema = resultsBlock.getSelectionDataSchema(); Assert.assertEquals(selectionDataSchema.size(), 4); Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6"); Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1"); Assert.assertEquals(selectionDataSchema.getColumnDataType(0), DataSchema.ColumnDataType.INT); Assert.assertEquals(selectionDataSchema.getColumnDataType(1), DataSchema.ColumnDataType.INT); selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult(); Assert.assertEquals(selectionResult.size(), 10);