public long getCommitTimeoutMS(String tableName) { long commitTimeoutMS = SegmentCompletionProtocol.getMaxSegmentCommitTimeMs(); if (_propertyStore == null) { return commitTimeoutMS; } TableConfig tableConfig = getRealtimeTableConfig(tableName); final Map<String, String> streamConfigs = tableConfig.getIndexingConfig().getStreamConfigs(); if (streamConfigs != null && streamConfigs.containsKey(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS)) { final String commitTimeoutSecondsStr = streamConfigs.get(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS); try { return TimeUnit.MILLISECONDS.convert(Integer.parseInt(commitTimeoutSecondsStr), TimeUnit.SECONDS); } catch (Exception e) { LOGGER.warn("Failed to parse flush size of {}", commitTimeoutSecondsStr, e); return commitTimeoutMS; } } return commitTimeoutMS; }
public MemoryEstimator(TableConfig tableConfig, File sampleCompletedSegment, long sampleSegmentConsumedSeconds) { _tableConfig = tableConfig; _sampleCompletedSegment = sampleCompletedSegment; _sampleSegmentConsumedSeconds = sampleSegmentConsumedSeconds; _sampleCompletedSegmentSizeBytes = FileUtils.sizeOfDirectory(_sampleCompletedSegment); try { _segmentMetadata = new SegmentMetadataImpl(_sampleCompletedSegment); } catch (Exception e) { throw new RuntimeException("Caught exception when reading segment index dir", e); } if (CollectionUtils.isNotEmpty(_tableConfig.getIndexingConfig().getNoDictionaryColumns())) { _noDictionaryColumns.addAll(_tableConfig.getIndexingConfig().getNoDictionaryColumns()); } if (CollectionUtils.isNotEmpty(_tableConfig.getIndexingConfig().getInvertedIndexColumns())) { _invertedIndexColumns.addAll(_tableConfig.getIndexingConfig().getInvertedIndexColumns()); } _avgMultiValues = getAvgMultiValues(); _tableDataDir = new File(TMP_DIR, _segmentMetadata.getTableName()); try { FileUtils.deleteDirectory(_tableDataDir); } catch (IOException e) { throw new RuntimeException("Exception in deleting directory " + _tableDataDir.getAbsolutePath(), e); } _tableDataDir.mkdir(); }
@Deprecated @PUT @Path("/tables/{tableName}/indexingConfigs") @ApiOperation(value = "Update table indexing configuration") @Produces(MediaType.APPLICATION_JSON) @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Table not found"), @ApiResponse(code = 500, message = "Server error updating configuration")}) public SuccessResponse updateIndexingConfig( @ApiParam(value = "Table name (without type)", required = true) @PathParam("tableName") String tableName, String body) { try { TableConfig tableConfig = TableConfig.fromJsonString(body); pinotHelixResourceManager.updateIndexingConfigFor(tableConfig.getTableName(), tableConfig.getTableType(), tableConfig.getIndexingConfig()); return new SuccessResponse("Updated indexing config for table " + tableName); } catch (IOException e) { String errStr = "Error converting request to table config for table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.BAD_REQUEST, e); } catch (Exception e) { String errStr = "Failed to update indexing config for table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); } } }
private SegmentPartitionMetadata getPartitionMetadataFromTableConfig(String tableName, int numPartitions, int partitionId) { Map<String, ColumnPartitionMetadata> partitionMetadataMap = new HashMap<>(); if (_propertyStore == null) { return null; } TableConfig tableConfig = getRealtimeTableConfig(tableName); SegmentPartitionMetadata partitionMetadata = null; SegmentPartitionConfig partitionConfig = tableConfig.getIndexingConfig().getSegmentPartitionConfig(); if (partitionConfig != null && partitionConfig.getColumnPartitionMap() != null && partitionConfig.getColumnPartitionMap().size() > 0) { Map<String, ColumnPartitionConfig> columnPartitionMap = partitionConfig.getColumnPartitionMap(); for (Map.Entry<String, ColumnPartitionConfig> entry : columnPartitionMap.entrySet()) { String column = entry.getKey(); ColumnPartitionConfig columnPartitionConfig = entry.getValue(); partitionMetadataMap.put(column, new ColumnPartitionMetadata(columnPartitionConfig.getFunctionName(), numPartitions, Collections.singleton(partitionId))); } partitionMetadata = new SegmentPartitionMetadata(partitionMetadataMap); } return partitionMetadata; }
public static IdealState buildInitialHighLevelRealtimeIdealStateFor(String realtimeTableName, TableConfig realtimeTableConfig, HelixManager helixManager, ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore, boolean enableBatchMessageMode) { RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(realtimeTableConfig); final List<String> realtimeInstances = HelixHelper.getInstancesWithTag(helixManager, realtimeTagConfig.getConsumingServerTag()); IdealState idealState = buildEmptyRealtimeIdealStateFor(realtimeTableName, 1, enableBatchMessageMode); if (realtimeInstances.size() % Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()) != 0) { throw new RuntimeException( "Number of instance in current tenant should be an integer multiples of the number of replications"); } setupInstanceConfigForHighLevelConsumer(realtimeTableName, realtimeInstances.size(), Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()), realtimeTableConfig.getIndexingConfig().getStreamConfigs(), zkHelixPropertyStore, realtimeInstances); return idealState; }
/** * Check table config for flush size. * * If flush size < 0, create a new DefaultFlushThresholdUpdater with default flush size * If flush size > 0, create a new DefaultFlushThresholdUpdater with given flush size. * If flush size == 0, create new SegmentSizeBasedFlushThresholdUpdater if not already created. Create only 1 per table, because we want to maintain tuning information for the table in the updater * @param realtimeTableConfig * @return */ public FlushThresholdUpdater getFlushThresholdUpdater(TableConfig realtimeTableConfig) { final String tableName = realtimeTableConfig.getTableName(); PartitionLevelStreamConfig streamConfig = new PartitionLevelStreamConfig(realtimeTableConfig.getIndexingConfig().getStreamConfigs()); final int tableFlushSize = streamConfig.getFlushThresholdRows(); final long desiredSegmentSize = streamConfig.getFlushSegmentDesiredSizeBytes(); if (tableFlushSize == 0) { return _flushThresholdUpdaterMap .computeIfAbsent(tableName, k -> new SegmentSizeBasedFlushThresholdUpdater(desiredSegmentSize)); } else { _flushThresholdUpdaterMap.remove(tableName); return new DefaultFlushThresholdUpdater(tableFlushSize); } }
@Override protected void processTable(String tableNameWithType) { CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); if (tableType == CommonConstants.Helix.TableType.REALTIME) { TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType); if (tableConfig == null) { LOGGER.warn("Failed to find table config for table: {}, skipping validation", tableNameWithType); return; } if (_updateRealtimeDocumentCount) { updateRealtimeDocumentCount(tableConfig); } Map<String, String> streamConfigMap = tableConfig.getIndexingConfig().getStreamConfigs(); StreamConfig streamConfig = new StreamConfig(streamConfigMap); if (streamConfig.hasLowLevelConsumerType()) { _llcRealtimeSegmentManager.ensureAllPartitionsConsuming(tableConfig); } } }
IndexingConfig mockIndexConfig = mock(IndexingConfig.class); when(mockIndexConfig.getStreamConfigs()).thenReturn(streamConfigMap); when(mockTableConfig.getIndexingConfig()).thenReturn(mockIndexConfig);
when(mockIndexConfig.getStreamConfigs()).thenReturn(streamConfigMap); when(mockTableConfig.getIndexingConfig()).thenReturn(mockIndexConfig); TenantConfig mockTenantConfig = mock(TenantConfig.class); when(mockTenantConfig.getServer()).thenReturn(serverTenant);
StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (!streamConfig.hasLowLevelConsumerType()) { LOGGER.info("Table {} does not have LLC and will have no partition assignment", tableNameWithType);
private void validateLoadedConfig(TableConfig config) { Assert.assertEquals(config.getTableName(), "mytable_OFFLINE"); Assert.assertEquals(config.getQuotaConfig().getStorage(), "125 GiB"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeValue(), "5"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeUnit(), "DAYS"); Assert.assertEquals(config.getTenantConfig().getBroker(), "foo"); Assert.assertEquals(config.getTenantConfig().getServer(), "bar"); Assert.assertEquals(config.getIndexingConfig().getSortedColumn(), Lists.newArrayList("foo")); } }
/** * Sets up a new table's segments metadata and returns the ideal state setup with initial segments * @param tableConfig * @param idealState * @param partitionCount * @return */ private IdealState setupTable(TableConfig tableConfig, IdealState idealState, int partitionCount) throws InvalidConfigException { final String tableNameWithType = tableConfig.getTableName(); if (!idealState.isEnabled()) { LOGGER.info("Skipping validation for disabled table {}", tableNameWithType); return idealState; } final StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); final long now = getCurrentTimeMs(); PartitionAssignment partitionAssignment = _streamPartitionAssignmentGenerator.generateStreamPartitionAssignment(tableConfig, partitionCount); Set<Integer> newPartitions = new HashSet<>(partitionCount); for (int partition = 0; partition < partitionCount; partition++) { newPartitions.add(partition); } OffsetCriteria offsetCriteria = streamConfig.getOffsetCriteria(); Set<String> consumingSegments = setupNewPartitions(tableConfig, streamConfig, offsetCriteria, partitionAssignment, newPartitions, now); RealtimeSegmentAssignmentStrategy segmentAssignmentStrategy = new ConsumingSegmentAssignmentStrategy(); Map<String, List<String>> assignments = segmentAssignmentStrategy.assign(consumingSegments, partitionAssignment); updateIdealState(idealState, null, consumingSegments, assignments); return idealState; }
private void checkTableConfigWithStarTreeConfig(TableConfig tableConfig, TableConfig tableConfigToCompare) throws Exception { // Check that the segment assignment configuration does exist. Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); Assert.assertNotNull(tableConfigToCompare.getIndexingConfig().getStarTreeIndexSpec()); // Check that the configurations are correct. StarTreeIndexSpec starTreeIndexSpec = tableConfigToCompare.getIndexingConfig().getStarTreeIndexSpec(); Set<String> dims = new HashSet<>(); dims.add("dims"); Assert.assertEquals(starTreeIndexSpec.getDimensionsSplitOrder(), Collections.singletonList("dim")); Assert.assertEquals(starTreeIndexSpec.getMaxLeafRecords(), 5); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationCardinalityThreshold(), 1); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationForDimensions(), dims); Assert.assertEquals(starTreeIndexSpec.getSkipStarNodeCreationForDimensions(), dims); starTreeIndexSpec = StarTreeIndexSpec.fromJsonString(starTreeIndexSpec.toJsonString()); Assert.assertEquals(starTreeIndexSpec.getDimensionsSplitOrder(), Collections.singletonList("dim")); Assert.assertEquals(starTreeIndexSpec.getMaxLeafRecords(), 5); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationCardinalityThreshold(), 1); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationForDimensions(), dims); Assert.assertEquals(starTreeIndexSpec.getSkipStarNodeCreationForDimensions(), dims); }
protected FakePinotLLCRealtimeSegmentManager(List<String> existingLLCSegments) { super(null, clusterName, null, null, null, CONTROLLER_CONF, new ControllerMetrics(new MetricsRegistry())); try { TableConfigCache mockCache = mock(TableConfigCache.class); TableConfig mockTableConfig = mock(TableConfig.class); IndexingConfig mockIndexingConfig = mock(IndexingConfig.class); when(mockTableConfig.getIndexingConfig()).thenReturn(mockIndexingConfig); when(mockIndexingConfig.getStreamConfigs()).thenReturn(getStreamConfigs()); when(mockCache.getTableConfig(anyString())).thenReturn(mockTableConfig); Field tableConfigCacheField = PinotLLCRealtimeSegmentManager.class.getDeclaredField("_tableConfigCache"); tableConfigCacheField.setAccessible(true); tableConfigCacheField.set(this, mockCache); HelixManager mockHelixManager = mock(HelixManager.class); _partitionAssignmentGenerator = new FakeStreamPartitionAssignmentGenerator(mockHelixManager); Field partitionAssignmentGeneratorField = PinotLLCRealtimeSegmentManager.class.getDeclaredField("_streamPartitionAssignmentGenerator"); partitionAssignmentGeneratorField.setAccessible(true); partitionAssignmentGeneratorField.set(this, _partitionAssignmentGenerator); } catch (Exception e) { Utils.rethrowException(e); } if (existingLLCSegments != null) { _existingLLCSegments = existingLLCSegments; } CONTROLLER_CONF.setControllerVipHost("vip"); CONTROLLER_CONF.setControllerPort("9000"); CONTROLLER_CONF.setDataDir(baseDir.toString()); _version = 0; _tableConfigStore = new TableConfigStore(); }
private void updateRealtimeDocumentCount(TableConfig tableConfig) { String realtimeTableName = tableConfig.getTableName(); List<RealtimeSegmentZKMetadata> metadataList = _pinotHelixResourceManager.getRealtimeSegmentMetadata(realtimeTableName); boolean countHLCSegments = true; // false if this table has ONLY LLC segments (i.e. fully migrated) StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (streamConfig.hasLowLevelConsumerType() && !streamConfig.hasHighLevelConsumerType()) { countHLCSegments = false; } // Update the gauge to contain the total document count in the segments _validationMetrics.updateTotalDocumentCountGauge(tableConfig.getTableName(), computeRealtimeTotalDocumentInSegments(metadataList, countHLCSegments)); }
final StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); final int partitionCount = getPartitionCount(streamConfig); HelixHelper.updateIdealState(_helixManager, tableNameWithType, new Function<IdealState, IdealState>() {
/** * * @param tableConfig * @param emptyIdealState may contain HLC segments if both HLC and LLC are configured */ public void setupNewTable(TableConfig tableConfig, IdealState emptyIdealState) throws InvalidConfigException { final StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); int partitionCount = getPartitionCount(streamConfig); List<String> currentSegments = getExistingSegments(tableConfig.getTableName()); // Make sure that there are no low-level segments existing. if (currentSegments != null) { for (String segment : currentSegments) { if (!SegmentName.isHighLevelConsumerSegmentName(segment)) { // For now, we don't support re-creating the low-level realtime segments throw new RuntimeException("Low-level segments already exist for table " + tableConfig.getTableType()); } } } _flushThresholdUpdateManager.clearFlushThresholdUpdater(tableConfig); if (!isConnected()) { throw new RuntimeException( "Lost zk connection while setting up new table " + tableConfig.getTableName() + " isConnected=" + isConnected()); } IdealState idealState = setupTable(tableConfig, emptyIdealState, partitionCount); setTableIdealState(tableConfig.getTableName(), idealState); }
protected void updateRealtimeTableConfig(String tablename, List<String> invertedIndexCols, List<String> bloomFilterCols) throws Exception { IndexingConfig config = _realtimeTableConfig.getIndexingConfig(); config.setInvertedIndexColumns(invertedIndexCols); config.setBloomFilterColumns(bloomFilterCols); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tablename), _realtimeTableConfig.toJSONConfigString()); }
public void setExistingTableConfig(TableConfig config, String tableNameWithType, TableType type) throws IOException { if (type == TableType.REALTIME) { ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); ensureRealtimeClusterIsSetUp(config, tableNameWithType, config.getIndexingConfig()); } else if (type == TableType.OFFLINE) { // Update replica group partition assignment to the property store if applicable updateReplicaGroupPartitionAssignment(config); ZKMetadataProvider.setOfflineTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); final String configReplication = config.getValidationConfig().getReplication(); if (configReplication != null && !config.getValidationConfig().getReplication() .equals(idealState.getReplicas())) { HelixHelper.updateIdealState(_helixZkManager, tableNameWithType, new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { idealState.setReplicas(configReplication); return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000L, 1.2f)); } } }
private TableConfig buildRealtimeTableConfig() throws Exception { // Create partition config Map<String, ColumnPartitionConfig> metadataMap = new HashMap<>(); metadataMap.put(PARTITION_COLUMN, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITION)); SegmentPartitionConfig partitionConfig = new SegmentPartitionConfig(metadataMap); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(REALTIME_TABLE_NAME) .setNumReplicas(NUM_REPLICA).build(); tableConfig.getValidationConfig().setReplicasPerPartition(Integer.toString(NUM_REPLICA)); tableConfig.getIndexingConfig().setSegmentPartitionConfig(partitionConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }