@Test public void testSerDe() throws IOException { IndexingConfig indexingConfig = new IndexingConfig(); indexingConfig.setLoadMode("MMAP"); indexingConfig.setAggregateMetrics(true); List<String> invertedIndexColumns = Arrays.asList("a", "b", "c"); indexingConfig.setInvertedIndexColumns(invertedIndexColumns); List<String> sortedColumn = Arrays.asList("d", "e", "f"); indexingConfig.setSortedColumn(sortedColumn); List<String> onHeapDictionaryColumns = Arrays.asList("x", "y", "z"); indexingConfig.setOnHeapDictionaryColumns(onHeapDictionaryColumns); List<String> bloomFilterColumns = Arrays.asList("a", "b"); indexingConfig.setBloomFilterColumns(bloomFilterColumns); Map<String, String> noDictionaryConfig = new HashMap<>(); noDictionaryConfig.put("a", "SNAPPY"); noDictionaryConfig.put("b", "PASS_THROUGH"); indexingConfig.setnoDictionaryConfig(noDictionaryConfig); indexingConfig = JsonUtils.stringToObject(JsonUtils.objectToString(indexingConfig), IndexingConfig.class); assertEquals(indexingConfig.getLoadMode(), "MMAP"); assertTrue(indexingConfig.isAggregateMetrics()); assertEquals(indexingConfig.getInvertedIndexColumns(), invertedIndexColumns); assertEquals(indexingConfig.getSortedColumn(), sortedColumn); assertEquals(indexingConfig.getOnHeapDictionaryColumns(), onHeapDictionaryColumns); assertEquals(indexingConfig.getBloomFilterColumns(), bloomFilterColumns); assertEquals(indexingConfig.getNoDictionaryConfig(), noDictionaryConfig); }
List<String> noDictionaryColumns = indexingConfig.getNoDictionaryColumns(); Map<String, String> noDictionaryColumnMap = indexingConfig.getNoDictionaryConfig(); _segmentPartitionConfig = indexingConfig.getSegmentPartitionConfig(); StarTreeIndexSpec starTreeIndexSpec = indexingConfig.getStarTreeIndexSpec(); if (starTreeIndexSpec != null) { enableStarTreeIndex(starTreeIndexSpec); List<StarTreeIndexConfig> starTreeIndexConfigs = indexingConfig.getStarTreeIndexConfigs(); if (starTreeIndexConfigs != null && !starTreeIndexConfigs.isEmpty()) { List<StarTreeV2BuilderConfig> starTreeV2BuilderConfigs = new ArrayList<>(starTreeIndexConfigs.size()); if (indexingConfig.isCreateInvertedIndexDuringSegmentGeneration()) { _invertedIndexCreationColumns = indexingConfig.getInvertedIndexColumns();
public long getCommitTimeoutMS(String tableName) { long commitTimeoutMS = SegmentCompletionProtocol.getMaxSegmentCommitTimeMs(); if (_propertyStore == null) { return commitTimeoutMS; } TableConfig tableConfig = getRealtimeTableConfig(tableName); final Map<String, String> streamConfigs = tableConfig.getIndexingConfig().getStreamConfigs(); if (streamConfigs != null && streamConfigs.containsKey(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS)) { final String commitTimeoutSecondsStr = streamConfigs.get(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS); try { return TimeUnit.MILLISECONDS.convert(Integer.parseInt(commitTimeoutSecondsStr), TimeUnit.SECONDS); } catch (Exception e) { LOGGER.warn("Failed to parse flush size of {}", commitTimeoutSecondsStr, e); return commitTimeoutMS; } } return commitTimeoutMS; }
private void extractFromTableConfig(@Nonnull TableConfig tableConfig) { IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); String tableReadMode = indexingConfig.getLoadMode(); if (tableReadMode != null) { _readMode = ReadMode.getEnum(tableReadMode); List<String> sortedColumns = indexingConfig.getSortedColumn(); if (sortedColumns != null) { _sortedColumns = sortedColumns; List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns(); if (invertedIndexColumns != null) { _invertedIndexColumns.addAll(invertedIndexColumns); List<String> bloomFilterColumns = indexingConfig.getBloomFilterColumns(); if (bloomFilterColumns != null) { _bloomFilterColumns.addAll(bloomFilterColumns); List<String> noDictionaryColumns = indexingConfig.getNoDictionaryColumns(); if (noDictionaryColumns != null) { _noDictionaryColumns.addAll(noDictionaryColumns); Map<String, String> noDictionaryConfig = indexingConfig.getNoDictionaryConfig(); if (noDictionaryConfig != null) { _noDictionaryConfig.putAll(noDictionaryConfig); List<String> onHeapDictionaryColumns = indexingConfig.getOnHeapDictionaryColumns(); if (onHeapDictionaryColumns != null) { _onHeapDictionaryColumns.addAll(onHeapDictionaryColumns);
public MemoryEstimator(TableConfig tableConfig, File sampleCompletedSegment, long sampleSegmentConsumedSeconds) { _tableConfig = tableConfig; _sampleCompletedSegment = sampleCompletedSegment; _sampleSegmentConsumedSeconds = sampleSegmentConsumedSeconds; _sampleCompletedSegmentSizeBytes = FileUtils.sizeOfDirectory(_sampleCompletedSegment); try { _segmentMetadata = new SegmentMetadataImpl(_sampleCompletedSegment); } catch (Exception e) { throw new RuntimeException("Caught exception when reading segment index dir", e); } if (CollectionUtils.isNotEmpty(_tableConfig.getIndexingConfig().getNoDictionaryColumns())) { _noDictionaryColumns.addAll(_tableConfig.getIndexingConfig().getNoDictionaryColumns()); } if (CollectionUtils.isNotEmpty(_tableConfig.getIndexingConfig().getInvertedIndexColumns())) { _invertedIndexColumns.addAll(_tableConfig.getIndexingConfig().getInvertedIndexColumns()); } _avgMultiValues = getAvgMultiValues(); _tableDataDir = new File(TMP_DIR, _segmentMetadata.getTableName()); try { FileUtils.deleteDirectory(_tableDataDir); } catch (IOException e) { throw new RuntimeException("Exception in deleting directory " + _tableDataDir.getAbsolutePath(), e); } _tableDataDir.mkdir(); }
IndexingConfig indexingConfig = new IndexingConfig(); indexingConfig.setLoadMode(_loadMode); indexingConfig.setSegmentFormatVersion(_segmentVersion); if (_sortedColumn != null) { indexingConfig.setSortedColumn(Collections.singletonList(_sortedColumn)); indexingConfig.setInvertedIndexColumns(_invertedIndexColumns); indexingConfig.setNoDictionaryColumns(_noDictionaryColumns); indexingConfig.setOnHeapDictionaryColumns(_onHeapDictionaryColumns); indexingConfig.setStreamConfigs(_streamConfigs); indexingConfig.setBloomFilterColumns(_bloomFilterColumns); StreamConsumptionConfig streamConsumptionConfig = new StreamConsumptionConfig(); streamConsumptionConfig.setStreamPartitionAssignmentStrategy(_streamPartitionAssignmentStrategy); indexingConfig.setStreamConsumptionConfig(streamConsumptionConfig);
_partitionLevelStreamConfig = new PartitionLevelStreamConfig(indexingConfig.getStreamConfigs()); _streamConsumerFactory = StreamConsumerFactoryProvider.create(_partitionLevelStreamConfig); _streamTopic = _partitionLevelStreamConfig.getTopicName(); _starTreeIndexSpec = indexingConfig.getStarTreeIndexSpec(); .setOffHeap(_isOffHeap).setMemoryManager(_memoryManager) .setStatsHistory(realtimeTableDataManager.getStatsHistory()) .setAggregateMetrics(indexingConfig.isAggregateMetrics()); makeStreamMetadataProvider("Starting"); SegmentPartitionConfig segmentPartitionConfig = indexingConfig.getSegmentPartitionConfig(); if (segmentPartitionConfig != null) { try {
Preconditions.checkNotNull(tableConfig); IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns(); boolean autoGeneratedInvertedIndex = indexingConfig.isAutoGeneratedInvertedIndex(); indexingConfig.setAutoGeneratedInvertedIndex(false); if (updateIndexConfig(tableNameWithType, tableConfig)) { LOGGER.info("Table: {}, removed auto-generated inverted index", tableNameWithType); indexingConfig.setInvertedIndexColumns(invertedIndexColumns); indexingConfig.setAutoGeneratedInvertedIndex(true); if (updateIndexConfig(tableNameWithType, tableConfig)) { LOGGER.info("Table: {}, added inverted index to columns: {}", tableNameWithType, invertedIndexColumns); indexingConfig.setAutoGeneratedInvertedIndex(false); if (updateIndexConfig(tableNameWithType, tableConfig)) { LOGGER.info("Table: {}, removed auto-generated inverted index", tableNameWithType);
@Test public void testSegmentPartitionConfig() throws IOException { int numColumns = 5; Map<String, ColumnPartitionConfig> expectedColumnPartitionMap = new HashMap<>(5); for (int i = 0; i < numColumns; i++) { expectedColumnPartitionMap.put("column_" + i, new ColumnPartitionConfig("function_" + i, i + 1)); } SegmentPartitionConfig expectedPartitionConfig = new SegmentPartitionConfig(expectedColumnPartitionMap); IndexingConfig expectedIndexingConfig = new IndexingConfig(); expectedIndexingConfig.setSegmentPartitionConfig(expectedPartitionConfig); IndexingConfig actualIndexingConfig = JsonUtils.stringToObject(JsonUtils.objectToString(expectedIndexingConfig), IndexingConfig.class); SegmentPartitionConfig actualPartitionConfig = actualIndexingConfig.getSegmentPartitionConfig(); Map<String, ColumnPartitionConfig> actualColumnPartitionMap = actualPartitionConfig.getColumnPartitionMap(); assertEquals(actualColumnPartitionMap.size(), expectedColumnPartitionMap.size()); for (String column : expectedColumnPartitionMap.keySet()) { assertEquals(actualPartitionConfig.getFunctionName(column), expectedPartitionConfig.getFunctionName(column)); assertEquals(actualPartitionConfig.getNumPartitions(column), expectedPartitionConfig.getNumPartitions(column)); } }
List<String> sortedColumn = _indexingConfig.getSortedColumn(); StarTreeIndexSpec starTreeIndexSpec = _indexingConfig.getStarTreeIndexSpec(); List<String> invertedIndexColumns = _indexingConfig.getInvertedIndexColumns();
protected void updateRealtimeTableConfig(String tablename, List<String> invertedIndexCols, List<String> bloomFilterCols) throws Exception { IndexingConfig config = _realtimeTableConfig.getIndexingConfig(); config.setInvertedIndexColumns(invertedIndexCols); config.setBloomFilterColumns(bloomFilterCols); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tablename), _realtimeTableConfig.toJSONConfigString()); }
/** * Helper function to trigger the segment creation * * TODO: Support all kinds of indexing (no dictionary) */ private void buildSegment(String outputPath, String tableName, String segmentName, RecordReader recordReader, IndexingConfig indexingConfig) throws Exception { SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(recordReader.getSchema()); segmentGeneratorConfig.setOutDir(outputPath); segmentGeneratorConfig.setTableName(tableName); segmentGeneratorConfig.setSegmentName(segmentName); if (indexingConfig != null) { segmentGeneratorConfig.setInvertedIndexCreationColumns(indexingConfig.getInvertedIndexColumns()); if (indexingConfig.getStarTreeIndexSpec() != null) { segmentGeneratorConfig.enableStarTreeIndex(indexingConfig.getStarTreeIndexSpec()); } } SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig, recordReader); driver.build(); }
expectedStarTreeSpec.setSkipStarNodeCreationForDimensions(expectedSkipStarNodeCreationForDimension); IndexingConfig expectedIndexingConfig = new IndexingConfig(); expectedIndexingConfig.setStarTreeIndexSpec(expectedStarTreeSpec); StarTreeIndexSpec actualStarTreeSpec = actualIndexingConfig.getStarTreeIndexSpec();
_streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (indexingConfig != null && indexingConfig.isAggregateMetrics()) { LOGGER.warn("Updating of metrics only supported for LLC consumer, ignoring.");
IndexingConfig indexingConfig = new IndexingConfig(); Map<String, ColumnPartitionConfig> partitionConfigMap = new HashMap<>(); partitionConfigMap.put(PARTITION_COLUMN, new ColumnPartitionConfig("modulo", totalPartitionNumber)); indexingConfig.setSegmentPartitionConfig(new SegmentPartitionConfig(partitionConfigMap));
private SegmentPartitionMetadata getPartitionMetadataFromTableConfig(String tableName, int numPartitions, int partitionId) { Map<String, ColumnPartitionMetadata> partitionMetadataMap = new HashMap<>(); if (_propertyStore == null) { return null; } TableConfig tableConfig = getRealtimeTableConfig(tableName); SegmentPartitionMetadata partitionMetadata = null; SegmentPartitionConfig partitionConfig = tableConfig.getIndexingConfig().getSegmentPartitionConfig(); if (partitionConfig != null && partitionConfig.getColumnPartitionMap() != null && partitionConfig.getColumnPartitionMap().size() > 0) { Map<String, ColumnPartitionConfig> columnPartitionMap = partitionConfig.getColumnPartitionMap(); for (Map.Entry<String, ColumnPartitionConfig> entry : columnPartitionMap.entrySet()) { String column = entry.getKey(); ColumnPartitionConfig columnPartitionConfig = entry.getValue(); partitionMetadataMap.put(column, new ColumnPartitionMetadata(columnPartitionConfig.getFunctionName(), numPartitions, Collections.singleton(partitionId))); } partitionMetadata = new SegmentPartitionMetadata(partitionMetadataMap); } return partitionMetadata; }
Assert.assertEquals(tableConfig.getIndexingConfig().getLoadMode(), "HEAP"); Assert.assertNull(tableConfig.getQuotaConfig()); Assert.assertEquals(tableConfig.getIndexingConfig().getLoadMode(), "HEAP"); Assert.assertNotNull(tableConfig.getQuotaConfig()); Assert.assertEquals(tableConfig.getQuotaConfig().getStorage(), "30G"); Assert.assertEquals(tableConfig.getIndexingConfig().getLoadMode(), "HEAP"); Assert.assertNotNull(tableConfig.getTenantConfig()); Assert.assertEquals(tableConfig.getTenantConfig().getServer(), "aServerTenant"); tableConfig.getIndexingConfig().getStreamConsumptionConfig().getStreamPartitionAssignmentStrategy(), "UniformStreamPartitionAssignment"); tableConfigBuilder.setStreamPartitionAssignmentStrategy("BalancedStreamPartitionAssignment").build(); Assert.assertEquals( tableConfig.getIndexingConfig().getStreamConsumptionConfig().getStreamPartitionAssignmentStrategy(), "BalancedStreamPartitionAssignment"); tableConfigToCompare.getIndexingConfig().getStreamConsumptionConfig().getStreamPartitionAssignmentStrategy(), "BalancedStreamPartitionAssignment"); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); Assert.assertEquals( tableConfigToCompare.getIndexingConfig().getStreamConsumptionConfig().getStreamPartitionAssignmentStrategy(), "BalancedStreamPartitionAssignment"); tableConfig.getIndexingConfig().setStarTreeIndexSpec(starTreeIndexSpec);
assertEquals(tableConfigurationObject.getOfflineTableConfig().getTableName(), "mytable_OFFLINE"); assertEquals(tableConfigurationObject.getOfflineTableConfig().getValidationConfig().getReplicationNumber(), 4); assertEquals(tableConfigurationObject.getOfflineTableConfig().getIndexingConfig().getLoadMode(), "MMAP"); assertEquals(tableConfigurationObject.getOfflineTableConfig().getTableName(), "mytable_OFFLINE"); assertEquals(tableConfigurationObject.getOfflineTableConfig().getValidationConfig().getReplicationNumber(), 3); assertEquals(tableConfigurationObject.getOfflineTableConfig().getIndexingConfig().getLoadMode(), "HEAP");
private TableConfig buildRealtimeTableConfig() throws Exception { // Create partition config Map<String, ColumnPartitionConfig> metadataMap = new HashMap<>(); metadataMap.put(PARTITION_COLUMN, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITION)); SegmentPartitionConfig partitionConfig = new SegmentPartitionConfig(metadataMap); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(REALTIME_TABLE_NAME) .setNumReplicas(NUM_REPLICA).build(); tableConfig.getValidationConfig().setReplicasPerPartition(Integer.toString(NUM_REPLICA)); tableConfig.getIndexingConfig().setSegmentPartitionConfig(partitionConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }
if (partitionColumn != null) { numColumnPartitions = tableConfig.getIndexingConfig().getSegmentPartitionConfig().getNumPartitions(partitionColumn);