private SegmentPartitionMetadata getPartitionMetadataFromTableConfig(String tableName, int numPartitions, int partitionId) { Map<String, ColumnPartitionMetadata> partitionMetadataMap = new HashMap<>(); if (_propertyStore == null) { return null; } TableConfig tableConfig = getRealtimeTableConfig(tableName); SegmentPartitionMetadata partitionMetadata = null; SegmentPartitionConfig partitionConfig = tableConfig.getIndexingConfig().getSegmentPartitionConfig(); if (partitionConfig != null && partitionConfig.getColumnPartitionMap() != null && partitionConfig.getColumnPartitionMap().size() > 0) { Map<String, ColumnPartitionConfig> columnPartitionMap = partitionConfig.getColumnPartitionMap(); for (Map.Entry<String, ColumnPartitionConfig> entry : columnPartitionMap.entrySet()) { String column = entry.getKey(); ColumnPartitionConfig columnPartitionConfig = entry.getValue(); partitionMetadataMap.put(column, new ColumnPartitionMetadata(columnPartitionConfig.getFunctionName(), numPartitions, Collections.singleton(partitionId))); } partitionMetadata = new SegmentPartitionMetadata(partitionMetadataMap); } return partitionMetadata; }
@Nullable public PartitionFunction getPartitionFunction(String column) { if (_segmentPartitionConfig == null) { return null; } String functionName = _segmentPartitionConfig.getFunctionName(column); int numPartitions = _segmentPartitionConfig.getNumPartitions(column); return (functionName != null) ? PartitionFunctionFactory.getPartitionFunction(functionName, numPartitions) : null; }
@Test public void testSegmentPartitionConfig() throws IOException { int numColumns = 5; Map<String, ColumnPartitionConfig> expectedColumnPartitionMap = new HashMap<>(5); for (int i = 0; i < numColumns; i++) { expectedColumnPartitionMap.put("column_" + i, new ColumnPartitionConfig("function_" + i, i + 1)); } SegmentPartitionConfig expectedPartitionConfig = new SegmentPartitionConfig(expectedColumnPartitionMap); IndexingConfig expectedIndexingConfig = new IndexingConfig(); expectedIndexingConfig.setSegmentPartitionConfig(expectedPartitionConfig); IndexingConfig actualIndexingConfig = JsonUtils.stringToObject(JsonUtils.objectToString(expectedIndexingConfig), IndexingConfig.class); SegmentPartitionConfig actualPartitionConfig = actualIndexingConfig.getSegmentPartitionConfig(); Map<String, ColumnPartitionConfig> actualColumnPartitionMap = actualPartitionConfig.getColumnPartitionMap(); assertEquals(actualColumnPartitionMap.size(), expectedColumnPartitionMap.size()); for (String column : expectedColumnPartitionMap.keySet()) { assertEquals(actualPartitionConfig.getFunctionName(column), expectedPartitionConfig.getFunctionName(column)); assertEquals(actualPartitionConfig.getNumPartitions(column), expectedPartitionConfig.getNumPartitions(column)); } }
SegmentPartitionConfig expectedConfig = new SegmentPartitionConfig(expectedMap); SegmentPartitionConfig actualConfig = SegmentPartitionConfig.fromJsonString(expectedConfig.toJsonString()); for (Map.Entry<String, ColumnPartitionConfig> entry : actualConfig.getColumnPartitionMap().entrySet()) { String partitionColumn = entry.getKey(); SegmentPartitionConfig.fromJsonString(jsonStringWithNewField).toJsonString());
.put(PARTITIONED_COLUMN_NAME, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITIONS)); SegmentPartitionConfig segmentPartitionConfig = new SegmentPartitionConfig(partitionFunctionMap); SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema);
/** * Returns the number of partitions for the specified column. * If segment partition config does not exist, returns {@link SegmentPartitionConfig#INVALID_NUM_PARTITIONS}. * * @param column Column for which to to return the number of partitions. * @return Number of partitions for the column. */ public int getNumPartitions(String column) { return (_segmentPartitionConfig != null) ? _segmentPartitionConfig.getNumPartitions(column) : SegmentPartitionConfig.INVALID_NUM_PARTITIONS; }
try { int nPartitions = _streamMetadataProvider.fetchPartitionCount(/*maxWaitTimeMs=*/5000L); segmentPartitionConfig.setNumPartitions(nPartitions); realtimeSegmentConfigBuilder.setSegmentPartitionConfig(segmentPartitionConfig); } catch (Exception e) {
private TableConfig buildRealtimeTableConfig() throws Exception { // Create partition config Map<String, ColumnPartitionConfig> metadataMap = new HashMap<>(); metadataMap.put(PARTITION_COLUMN, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITION)); SegmentPartitionConfig partitionConfig = new SegmentPartitionConfig(metadataMap); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(REALTIME_TABLE_NAME) .setNumReplicas(NUM_REPLICA).build(); tableConfig.getValidationConfig().setReplicasPerPartition(Integer.toString(NUM_REPLICA)); tableConfig.getIndexingConfig().setSegmentPartitionConfig(partitionConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }
if (partitionColumn != null) { numColumnPartitions = tableConfig.getIndexingConfig().getSegmentPartitionConfig().getNumPartitions(partitionColumn);
public RealtimeSegmentStatsContainer(MutableSegmentImpl realtimeSegment, RealtimeSegmentRecordReader realtimeSegmentRecordReader) { _realtimeSegment = realtimeSegment; _realtimeSegmentRecordReader = realtimeSegmentRecordReader; SegmentPartitionConfig segmentPartitionConfig = _realtimeSegment.getSegmentPartitionConfig(); // Create all column statistics for (String columnName : realtimeSegment.getPhysicalColumnNames()) { ColumnDataSource dataSource = realtimeSegment.getDataSource(columnName); if (dataSource.getDataSourceMetadata().hasDictionary()) { _columnStatisticsMap.put(columnName, new RealtimeColumnStatistics(realtimeSegment.getDataSource(columnName), _realtimeSegmentRecordReader.getSortedDocIdIterationOrder(), (segmentPartitionConfig == null) ? null : segmentPartitionConfig.getColumnPartitionMap().get(columnName))); } else { _columnStatisticsMap.put(columnName, new RealtimeNoDictionaryColStatistics(dataSource)); } } }
Map<String, ColumnPartitionConfig> partitionConfigMap = new HashMap<>(); partitionConfigMap.put(PARTITION_COLUMN, new ColumnPartitionConfig("modulo", totalPartitionNumber)); indexingConfig.setSegmentPartitionConfig(new SegmentPartitionConfig(partitionConfigMap));
driver.build(); if (segmentPartitionConfig != null && segmentPartitionConfig.getColumnPartitionMap() != null) { Map<String, ColumnPartitionConfig> columnPartitionMap = segmentPartitionConfig.getColumnPartitionMap(); for (String columnName : columnPartitionMap.keySet()) { int numPartitions = driver.getSegmentStats().getColumnProfileFor(columnName).getPartitions().size();