private int getOrCreateDocId(Map<String, Object> dictIdMap) { if (!_aggregateMetrics) { return _numDocsIndexed; } int i = 0; int[] dictIds = new int[_numKeyColumns]; // dimensions + time column. for (String column : _schema.getDimensionNames()) { dictIds[i++] = (Integer) dictIdMap.get(column); } String timeColumnName = _schema.getTimeColumnName(); if (timeColumnName != null) { dictIds[i] = (Integer) dictIdMap.get(timeColumnName); } return _recordIdMap.put(new FixedIntArray(dictIds)); }
for (String dimension : schema.getDimensionNames()) { if (noDictionaryColumns.contains(dimension)) { _logger
List<String> dimensionNames = tableSchema.getDimensionNames(); if (dimensionNames.size() == 0) { LOGGER.info("Table: {}, skip adding inverted index because it does not have any dimension column",
Schema schema = segmentMetadata.getSchema(); ArrayList<String> dimensionsSplitOrder = new ArrayList<>(schema.getDimensionNames()); dimensionsSplitOrder.removeAll(starTreeMetadata.getSkipMaterializationForDimensions());
throws Exception { List<String> allDimensions = new ArrayList<>(_schema.getDimensionNames()); Collections.shuffle(allDimensions); for (int i = 0; i < NUM_STAR_TREE_DIMENSIONS; i++) {
_statsHistory = config.getStatsHistory(); _segmentPartitionConfig = config.getSegmentPartitionConfig(); _numKeyColumns = _schema.getDimensionNames().size() + 1;
public void addColumnMinMaxValue() throws Exception { Preconditions.checkState(_columnMinMaxValueGeneratorMode != ColumnMinMaxValueGeneratorMode.NONE); Schema schema = _segmentMetadata.getSchema(); // Process time column String timeColumnName = schema.getTimeColumnName(); if (timeColumnName != null) { addColumnMinMaxValueForColumn(timeColumnName); } if (_columnMinMaxValueGeneratorMode == ColumnMinMaxValueGeneratorMode.TIME) { saveMetadata(); return; } // Process dimension columns for (String dimensionColumnName : schema.getDimensionNames()) { addColumnMinMaxValueForColumn(dimensionColumnName); } if (_columnMinMaxValueGeneratorMode == ColumnMinMaxValueGeneratorMode.NON_METRIC) { saveMetadata(); return; } // Process metric columns for (String metricColumnName : schema.getMetricNames()) { addColumnMinMaxValueForColumn(metricColumnName); } saveMetadata(); }
private File buildSegment(Schema schema, boolean isSimpleDate) throws Exception { SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema); config.setRawIndexCreationColumns(schema.getDimensionNames()); config.setOutDir(SEGMENT_DIR_NAME); config.setSegmentName(SEGMENT_NAME); config.setTimeColumnName(TIME_COL_NAME); if (isSimpleDate) { config.setSimpleDateFormat(TIME_COL_FORMAT); } List<GenericRow> rows = new ArrayList<>(NUM_ROWS); for (int i = 0; i < NUM_ROWS; i++) { HashMap<String, Object> map = new HashMap<>(); for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) { Object value; value = getRandomValueForColumn(fieldSpec, isSimpleDate); map.put(fieldSpec.getName(), value); } GenericRow genericRow = new GenericRow(); genericRow.init(map); rows.add(genericRow); } SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl(); driver.init(config, new GenericRowRecordReader(rows, schema)); driver.build(); driver.getOutputDirectory().deleteOnExit(); return driver.getOutputDirectory(); }
throws Exception { SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema); config.setRawIndexCreationColumns(schema.getDimensionNames());