private void validateLoadedConfig(TableConfig config) { Assert.assertEquals(config.getTableName(), "mytable_OFFLINE"); Assert.assertEquals(config.getQuotaConfig().getStorage(), "125 GiB"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeValue(), "5"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeUnit(), "DAYS"); Assert.assertEquals(config.getTenantConfig().getBroker(), "foo"); Assert.assertEquals(config.getTenantConfig().getServer(), "bar"); Assert.assertEquals(config.getIndexingConfig().getSortedColumn(), Lists.newArrayList("foo")); } }
@Deprecated @PUT @Path("/tables/{tableName}/segmentConfigs") @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "Update segments configuration", notes = "Updates segmentsConfig section (validation and retention) of a table") @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Table not found"), @ApiResponse(code = 500, message = "Internal server error")}) public SuccessResponse put(@ApiParam(value = "Table name", required = true) @PathParam("tableName") String tableName, String requestBody) { try { TableConfig tableConfig = TableConfig.fromJsonString(requestBody); pinotHelixResourceManager .updateSegmentsValidationAndRetentionConfigFor(tableConfig.getTableName(), tableConfig.getTableType(), tableConfig.getValidationConfig()); return new SuccessResponse("Update segmentsConfig for table: " + tableName); } catch (IOException e) { metrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_TABLE_SCHEMA_UPDATE_ERROR, 1L); throw new ControllerApplicationException(LOGGER, String.format("Invalid json while updating segments config for table: %s", tableName), Response.Status.BAD_REQUEST, e); } catch (Exception e) { metrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_TABLE_SCHEMA_UPDATE_ERROR, 1L); throw new ControllerApplicationException(LOGGER, String.format("Failed to update segments config for table: %s", tableName), Response.Status.INTERNAL_SERVER_ERROR, e); } } }
private boolean updateIndexConfig(String tableName, TableConfig tableConfig) throws Exception { String request = ControllerRequestURLBuilder.baseUrl("http://" + _controllerAddress).forTableUpdateIndexingConfigs(tableName); HttpURLConnection httpURLConnection = (HttpURLConnection) new URL(request).openConnection(); httpURLConnection.setDoOutput(true); httpURLConnection.setRequestMethod("PUT"); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(httpURLConnection.getOutputStream(), "UTF-8")); writer.write(tableConfig.toJSONConfigString()); writer.flush(); BufferedReader reader = new BufferedReader(new InputStreamReader(httpURLConnection.getInputStream(), "UTF-8")); return reader.readLine().equals("done"); }
public List<String> getAllTenantTables() throws Exception { String tableConfigPath = "/CONFIGS/TABLE"; List<ZNRecord> tableConfigs = _propertyStore.getChildren(tableConfigPath, null, 0); List<String> tables = new ArrayList<>(128); for (ZNRecord znRecord : tableConfigs) { TableConfig tableConfig = TableConfig.fromZnRecord(znRecord); if (tableConfig.getTenantConfig().getServer().equals(_tenantName)) { tables.add(tableConfig.getTableName()); } } return tables; }
@Override public void init(Configuration configuration, TableConfig tableConfig, ZkHelixPropertyStore<ZNRecord> propertyStore, BrokerMetrics brokerMetrics) { _tableName = tableConfig.getTableName(); _brokerMetrics = brokerMetrics; // Enable dynamic routing when the config is explicitly set RoutingConfig routingConfig = tableConfig.getRoutingConfig(); if (routingConfig != null) { Map<String, String> routingOption = routingConfig.getRoutingTableBuilderOptions(); _enableDynamicComputing = Boolean.valueOf(routingOption.get(RoutingConfig.ENABLE_DYNAMIC_COMPUTING_KEY)); if (_enableDynamicComputing) { LOGGER.info("Dynamic routing table computation is enabled for table {}", _tableName); } } }
@Deprecated @PUT @Path("/tables/{tableName}/metadataConfigs") @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "Update table metadata", notes = "Updates table configuration") @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error"), @ApiResponse(code = 404, message = "Table not found")}) public SuccessResponse updateTableMetadata(@PathParam("tableName") String tableName, String requestBody) { try { TableConfig tableConfig = TableConfig.fromJsonString(requestBody); pinotHelixResourceManager.updateMetadataConfigFor(tableConfig.getTableName(), tableConfig.getTableType(), tableConfig.getCustomConfig()); return new SuccessResponse("Successfully updated " + tableName + " configuration"); } catch (Exception e) { String errStr = "Error while updating table configuration, table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); } } }
Assert.assertEquals(tableConfig.getTableName(), "myTable_OFFLINE"); Assert.assertEquals(tableConfig.getTableType(), TableType.OFFLINE); Assert.assertEquals(tableConfig.getIndexingConfig().getLoadMode(), "HEAP"); Assert.assertNull(tableConfig.getQuotaConfig()); JsonNode jsonTableConfig = TableConfig.toJSONConfig(tableConfig); TableConfig tableConfigToCompare = TableConfig.fromJSONConfig(jsonTableConfig); Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); Assert.assertNull(tableConfigToCompare.getQuotaConfig()); Assert.assertNull(tableConfigToCompare.getValidationConfig().getReplicaGroupStrategyConfig()); Assert.assertNull(tableConfigToCompare.getValidationConfig().getHllConfig()); ZNRecord znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); Assert.assertNull(tableConfigToCompare.getQuotaConfig()); Assert.assertNull(tableConfig.getValidationConfig().getReplicaGroupStrategyConfig()); Assert.assertNull(tableConfigToCompare.getValidationConfig().getHllConfig()); Assert.assertEquals(tableConfig.getTableName(), "myTable_OFFLINE"); Assert.assertEquals(tableConfig.getTableType(), TableType.OFFLINE); Assert.assertEquals(tableConfig.getIndexingConfig().getLoadMode(), "HEAP"); Assert.assertNotNull(tableConfig.getQuotaConfig()); Assert.assertEquals(tableConfig.getQuotaConfig().getStorage(), "30G"); Assert.assertNull(tableConfig.getQuotaConfig().getMaxQueriesPerSecond()); Assert.assertNotNull(tableConfig.getQuotaConfig()); Assert.assertNotNull(tableConfig.getQuotaConfig().getMaxQueriesPerSecond());
private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs, long minStartTime, long maxEndTime) throws Exception { String tableName = tableConfig.getTableName(); // Fetch time related configurations from schema and table config. String pushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency(); String timeColumnType = tableConfig.getValidationConfig().getTimeType(); String pushType = tableConfig.getValidationConfig().getSegmentPushType(); String timeFormat = schema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeFormat(); // Generate the final segment name using segment name generator NormalizedDateSegmentNameGenerator segmentNameGenerator = new NormalizedDateSegmentNameGenerator(tableName, DEFAULT_SEQUENCE_ID, timeColumnType, pushFrequency, pushType, null, null, timeFormat); return segmentNameGenerator.generateSegmentName(minStartTime, maxEndTime); }
private TableConfig makeTableConfig(String tableName, int nReplicas, String consumerTypesCSV) { TableConfig mockTableConfig = mock(TableConfig.class); when(mockTableConfig.getTableName()).thenReturn(tableName); SegmentsValidationAndRetentionConfig mockValidationConfig = mock(SegmentsValidationAndRetentionConfig.class); when(mockValidationConfig.getReplicasPerPartition()).thenReturn(Integer.toString(nReplicas)); when(mockValidationConfig.getReplicasPerPartitionNumber()).thenReturn(nReplicas); when(mockTableConfig.getValidationConfig()).thenReturn(mockValidationConfig); CommonConstants.Helix.TableType tableTypeFromTableName = TableNameBuilder.getTableTypeFromTableName(tableName); when(mockTableConfig.getTableType()).thenReturn(tableTypeFromTableName); IndexingConfig mockIndexConfig = mock(IndexingConfig.class); when(mockIndexConfig.getStreamConfigs()).thenReturn(streamConfigMap); when(mockTableConfig.getIndexingConfig()).thenReturn(mockIndexConfig);
@Deprecated @PUT @Path("/tables/{tableName}/indexingConfigs") @ApiOperation(value = "Update table indexing configuration") @Produces(MediaType.APPLICATION_JSON) @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Table not found"), @ApiResponse(code = 500, message = "Server error updating configuration")}) public SuccessResponse updateIndexingConfig( @ApiParam(value = "Table name (without type)", required = true) @PathParam("tableName") String tableName, String body) { try { TableConfig tableConfig = TableConfig.fromJsonString(body); pinotHelixResourceManager.updateIndexingConfigFor(tableConfig.getTableName(), tableConfig.getTableType(), tableConfig.getIndexingConfig()); return new SuccessResponse("Updated indexing config for table " + tableName); } catch (IOException e) { String errStr = "Error converting request to table config for table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.BAD_REQUEST, e); } catch (Exception e) { String errStr = "Failed to update indexing config for table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); } } }
private void updateRealtimeDocumentCount(TableConfig tableConfig) { String realtimeTableName = tableConfig.getTableName(); List<RealtimeSegmentZKMetadata> metadataList = _pinotHelixResourceManager.getRealtimeSegmentMetadata(realtimeTableName); boolean countHLCSegments = true; // false if this table has ONLY LLC segments (i.e. fully migrated) StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (streamConfig.hasLowLevelConsumerType() && !streamConfig.hasHighLevelConsumerType()) { countHLCSegments = false; } // Update the gauge to contain the total document count in the segments _validationMetrics.updateTotalDocumentCountGauge(tableConfig.getTableName(), computeRealtimeTotalDocumentInSegments(metadataList, countHLCSegments)); }
/** * If realtime table and includeConsuming=true, rebalance consuming segments. NewPartitionAssignment will be used only * in this case. Always rebalance completed (online) segments, NewPartitionAssignment unused in this case * @param idealState old ideal state * @param tableConfig table config of table tor rebalance * @param rebalanceUserConfig custom user configs for specific rebalance strategies * @param newPartitionAssignment new rebalanced partition assignments as part of the resource rebalance * @return */ @Override public IdealState getRebalancedIdealState(IdealState idealState, TableConfig tableConfig, Configuration rebalanceUserConfig, PartitionAssignment newPartitionAssignment) { String tableNameWithType = tableConfig.getTableName(); CommonConstants.Helix.TableType tableType = tableConfig.getTableType(); LOGGER.info("Rebalancing ideal state for table {}", tableNameWithType); // get target num replicas int targetNumReplicas; if (tableType.equals(CommonConstants.Helix.TableType.REALTIME)) { String replicasString = tableConfig.getValidationConfig().getReplicasPerPartition(); try { targetNumReplicas = Integer.parseInt(replicasString); } catch (Exception e) { throw new RuntimeException("Invalid value for replicasPerPartition:'" + replicasString + "'", e); } } else { targetNumReplicas = Integer.parseInt(tableConfig.getValidationConfig().getReplication()); } return rebalanceIdealState(idealState, tableConfig, targetNumReplicas, rebalanceUserConfig, newPartitionAssignment); }
Configuration rebalanceUserConfig) throws InvalidConfigException { String tableNameWithType = tableConfig.getTableName(); PartitionAssignment newPartitionAssignment = new PartitionAssignment(tableNameWithType); if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (!streamConfig.hasLowLevelConsumerType()) { LOGGER.info("Table {} does not have LLC and will have no partition assignment", tableNameWithType); LOGGER.info("Rebalancing stream partition assignment for table {}", tableConfig.getTableName()); } else { LOGGER.info("includeConsuming = false. No need to rebalance partition assignment for {}", tableConfig.getTableType());
private void ensureMinReplicas(TableConfig config) { SegmentsValidationAndRetentionConfig segmentsConfig = config.getValidationConfig(); int configMinReplication = _controllerConf.getDefaultTableMinReplicas(); boolean verifyReplicasPerPartition = false; boolean verifyReplication = true; if (config.getTableType() == CommonConstants.Helix.TableType.REALTIME) { StreamConfig streamConfig; try { streamConfig = new StreamConfig(config.getIndexingConfig().getStreamConfigs()); } catch (Exception e) { String errorMsg = String.format("Invalid tableIndexConfig or streamConfig: %s", e.getMessage());
public void setExistingTableConfig(TableConfig config, String tableNameWithType, TableType type) throws IOException { if (type == TableType.REALTIME) { ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); ensureRealtimeClusterIsSetUp(config, tableNameWithType, config.getIndexingConfig()); } else if (type == TableType.OFFLINE) { // Update replica group partition assignment to the property store if applicable updateReplicaGroupPartitionAssignment(config); ZKMetadataProvider.setOfflineTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); final String configReplication = config.getValidationConfig().getReplication(); if (configReplication != null && !config.getValidationConfig().getReplication() .equals(idealState.getReplicas())) { HelixHelper.updateIdealState(_helixZkManager, tableNameWithType, new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { idealState.setReplicas(configReplication); return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000L, 1.2f)); } } }
@Override public void init(TableConfig tableConfig, ZkHelixPropertyStore<ZNRecord> propertyStore) { _tableNameWithType = tableConfig.getTableName(); _propertyStore = propertyStore; }
@Override public void init(Configuration configuration, TableConfig tableConfig, ZkHelixPropertyStore<ZNRecord> propertyStore, BrokerMetrics brokerMetrics) { super.init(configuration, tableConfig, propertyStore, brokerMetrics); _numReplicas = Integer.valueOf(tableConfig.getValidationConfig().getReplicasPerPartition()); }
QuotaConfig quotaConfig = _tableConfig.getQuotaConfig(); int numReplicas = _tableConfig.getValidationConfig().getReplicationNumber(); final String tableName = _tableConfig.getTableName();
String znRecordId = tableConfigZnRecord.getId(); if (TableNameBuilder.getTableTypeFromTableName(znRecordId) == TableType.REALTIME) { TableConfig tableConfig = TableConfig.fromZnRecord(tableConfigZnRecord); StreamConfig metadata = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (metadata.hasHighLevelConsumerType()) { String realtimeTable = tableConfig.getTableName(); String realtimeSegmentsPathForTable = _propertyStorePath + SEGMENTS_PATH + "/" + realtimeTable; RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider .getRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), tableConfig.getTableName(), segmentName); if (realtimeSegmentZKMetadata == null) {
String tableName = "updateTC"; String tableJSONConfigString = _offlineBuilder.setTableName(tableName).setNumReplicas(2).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeValue(), "5"); Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeUnit(), "DAYS"); tableConfig.getValidationConfig().setRetentionTimeUnit("HOURS"); tableConfig.getValidationConfig().setRetentionTimeValue("10"); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString())); Assert.assertTrue(jsonResponse.has("status")); Assert.assertEquals(modifiedConfig.getValidationConfig().getRetentionTimeUnit(), "HOURS"); Assert.assertEquals(modifiedConfig.getValidationConfig().getRetentionTimeValue(), "10"); tableJSONConfigString = _realtimeBuilder.setTableName(tableName).setNumReplicas(2).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); tableConfig = getTableConfig(tableName, "REALTIME"); Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeValue(), "5"); Assert.assertEquals(tableConfig.getValidationConfig().getRetentionTimeUnit(), "DAYS"); Assert.assertNull(tableConfig.getQuotaConfig()); tableConfig.setQuotaConfig(quota); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString()); modifiedConfig = getTableConfig(tableName, "REALTIME"); Assert.assertNotNull(modifiedConfig.getQuotaConfig()); Assert.assertEquals(modifiedConfig.getQuotaConfig().getStorage(), "10G");