/** * Returns the partition function for the given column, null if there isn't one. * * @param column Column for which to return the partition function. * @return Partition function for the column. */ @Nullable public String getFunctionName(@Nonnull String column) { ColumnPartitionConfig columnPartitionConfig = _columnPartitionMap.get(column); return (columnPartitionConfig != null) ? columnPartitionConfig.getFunctionName() : null; }
/** * Returns the number of partitions for the specified column. * Returns {@link #INVALID_NUM_PARTITIONS} if it does not exist for the column. * * @param column Column for which to get number of partitions. * @return Number of partitions of the column. */ public int getNumPartitions(String column) { ColumnPartitionConfig config = _columnPartitionMap.get(column); return (config != null) ? config.getNumPartitions() : INVALID_NUM_PARTITIONS; }
/** * Set the number of partitions for all columns. * * @param numPartitions Number of partitions. */ @JsonIgnore public void setNumPartitions(int numPartitions) { for (ColumnPartitionConfig columnPartitionConfig : _columnPartitionMap.values()) { columnPartitionConfig.setNumPartitions(numPartitions); } }
String partitionColumn = "column_" + i; String partitionFunction = "function_" + i; expectedMap.put(partitionColumn, new ColumnPartitionConfig(partitionFunction, i + 1)); ColumnPartitionConfig actualColumnConfig = entry.getValue(); assertEquals(actualColumnConfig.getFunctionName(), expectedColumnConfig.getFunctionName());
@Test public void testSegmentPartitionConfig() throws IOException { int numColumns = 5; Map<String, ColumnPartitionConfig> expectedColumnPartitionMap = new HashMap<>(5); for (int i = 0; i < numColumns; i++) { expectedColumnPartitionMap.put("column_" + i, new ColumnPartitionConfig("function_" + i, i + 1)); } SegmentPartitionConfig expectedPartitionConfig = new SegmentPartitionConfig(expectedColumnPartitionMap); IndexingConfig expectedIndexingConfig = new IndexingConfig(); expectedIndexingConfig.setSegmentPartitionConfig(expectedPartitionConfig); IndexingConfig actualIndexingConfig = JsonUtils.stringToObject(JsonUtils.objectToString(expectedIndexingConfig), IndexingConfig.class); SegmentPartitionConfig actualPartitionConfig = actualIndexingConfig.getSegmentPartitionConfig(); Map<String, ColumnPartitionConfig> actualColumnPartitionMap = actualPartitionConfig.getColumnPartitionMap(); assertEquals(actualColumnPartitionMap.size(), expectedColumnPartitionMap.size()); for (String column : expectedColumnPartitionMap.keySet()) { assertEquals(actualPartitionConfig.getFunctionName(column), expectedPartitionConfig.getFunctionName(column)); assertEquals(actualPartitionConfig.getNumPartitions(column), expectedPartitionConfig.getNumPartitions(column)); } }
public RealtimeColumnStatistics(ColumnDataSource dataSource, int[] sortedDocIdIterationOrder, ColumnPartitionConfig columnPartitionConfig) { _dataSource = dataSource; _sortedDocIdIterationOrder = sortedDocIdIterationOrder; _dictionaryReader = (MutableDictionary) dataSource.getDictionary(); _block = dataSource.nextBlock(); if (columnPartitionConfig != null) { String functionName = columnPartitionConfig.getFunctionName(); _numPartitions = columnPartitionConfig.getNumPartitions(); _partitionFunction = (functionName != null) ? PartitionFunctionFactory.getPartitionFunction(functionName, _numPartitions) : null; if (_partitionFunction != null) { // Iterate over the dictionary to check the partitioning _partitions = new HashSet<>(); int length = _dictionaryReader.length(); for (int i = 0; i < length; i++) { _partitions.add(_partitionFunction.getPartition(_dictionaryReader.get(i))); } } } }
.put(PARTITIONED_COLUMN_NAME, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITIONS));
private TableConfig buildRealtimeTableConfig() throws Exception { // Create partition config Map<String, ColumnPartitionConfig> metadataMap = new HashMap<>(); metadataMap.put(PARTITION_COLUMN, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITION)); SegmentPartitionConfig partitionConfig = new SegmentPartitionConfig(metadataMap); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(REALTIME_TABLE_NAME) .setNumReplicas(NUM_REPLICA).build(); tableConfig.getValidationConfig().setReplicasPerPartition(Integer.toString(NUM_REPLICA)); tableConfig.getIndexingConfig().setSegmentPartitionConfig(partitionConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }
private SegmentPartitionMetadata getPartitionMetadataFromTableConfig(String tableName, int numPartitions, int partitionId) { Map<String, ColumnPartitionMetadata> partitionMetadataMap = new HashMap<>(); if (_propertyStore == null) { return null; } TableConfig tableConfig = getRealtimeTableConfig(tableName); SegmentPartitionMetadata partitionMetadata = null; SegmentPartitionConfig partitionConfig = tableConfig.getIndexingConfig().getSegmentPartitionConfig(); if (partitionConfig != null && partitionConfig.getColumnPartitionMap() != null && partitionConfig.getColumnPartitionMap().size() > 0) { Map<String, ColumnPartitionConfig> columnPartitionMap = partitionConfig.getColumnPartitionMap(); for (Map.Entry<String, ColumnPartitionConfig> entry : columnPartitionMap.entrySet()) { String column = entry.getKey(); ColumnPartitionConfig columnPartitionConfig = entry.getValue(); partitionMetadataMap.put(column, new ColumnPartitionMetadata(columnPartitionConfig.getFunctionName(), numPartitions, Collections.singleton(partitionId))); } partitionMetadata = new SegmentPartitionMetadata(partitionMetadataMap); } return partitionMetadata; }
partitionConfigMap.put(PARTITION_COLUMN, new ColumnPartitionConfig("modulo", totalPartitionNumber)); indexingConfig.setSegmentPartitionConfig(new SegmentPartitionConfig(partitionConfigMap));