@Deprecated @PUT @Path("/tables/{tableName}/metadataConfigs") @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "Update table metadata", notes = "Updates table configuration") @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 500, message = "Internal server error"), @ApiResponse(code = 404, message = "Table not found")}) public SuccessResponse updateTableMetadata(@PathParam("tableName") String tableName, String requestBody) { try { TableConfig tableConfig = TableConfig.fromJsonString(requestBody); pinotHelixResourceManager.updateMetadataConfigFor(tableConfig.getTableName(), tableConfig.getTableType(), tableConfig.getCustomConfig()); return new SuccessResponse("Successfully updated " + tableName + " configuration"); } catch (Exception e) { String errStr = "Error while updating table configuration, table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); } } }
/** * Rebalance the segments for replica group tables. * @param idealState old ideal state * @param tableConfig table config of table tor rebalance * @param rebalanceUserConfig custom user configs for specific rebalance strategies * @param newPartitionAssignment new rebalaned partition assignments as part of the resource rebalance * @return a rebalanced idealstate */ @Override public IdealState getRebalancedIdealState(IdealState idealState, TableConfig tableConfig, Configuration rebalanceUserConfig, PartitionAssignment newPartitionAssignment) throws InvalidConfigException { // Currently, only offline table is supported if (tableConfig.getTableType() == CommonConstants.Helix.TableType.REALTIME) { throw new InvalidConfigException("Realtime table is not supported by replica group rebalancer"); } ReplicaGroupPartitionAssignment newReplicaGroupPartitionAssignment = (ReplicaGroupPartitionAssignment) newPartitionAssignment; return rebalanceSegments(idealState, tableConfig, newReplicaGroupPartitionAssignment); }
@Deprecated @PUT @Path("/tables/{tableName}/indexingConfigs") @ApiOperation(value = "Update table indexing configuration") @Produces(MediaType.APPLICATION_JSON) @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Table not found"), @ApiResponse(code = 500, message = "Server error updating configuration")}) public SuccessResponse updateIndexingConfig( @ApiParam(value = "Table name (without type)", required = true) @PathParam("tableName") String tableName, String body) { try { TableConfig tableConfig = TableConfig.fromJsonString(body); pinotHelixResourceManager.updateIndexingConfigFor(tableConfig.getTableName(), tableConfig.getTableType(), tableConfig.getIndexingConfig()); return new SuccessResponse("Updated indexing config for table " + tableName); } catch (IOException e) { String errStr = "Error converting request to table config for table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.BAD_REQUEST, e); } catch (Exception e) { String errStr = "Failed to update indexing config for table: " + tableName; throw new ControllerApplicationException(LOGGER, errStr, Response.Status.INTERNAL_SERVER_ERROR, e); } } }
@POST @Path("/tables/validate") @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "Validate table config for a table", notes = "This API returns the table config that matches the one you get from 'GET /tables/{tableName}'." + " This allows us to validate table config before apply.") public String checkTableConfig(String tableConfigStr) { try { ObjectNode tableConfigValidateStr = JsonUtils.newObjectNode(); TableConfig tableConfig = TableConfig.fromJsonString(tableConfigStr); if (tableConfig.getTableType() == CommonConstants.Helix.TableType.OFFLINE) { tableConfigValidateStr .set(CommonConstants.Helix.TableType.OFFLINE.name(), TableConfig.toJSONConfig(tableConfig)); } else { tableConfigValidateStr .set(CommonConstants.Helix.TableType.REALTIME.name(), TableConfig.toJSONConfig(tableConfig)); } return tableConfigValidateStr.toString(); } catch (Exception e) { throw new ControllerApplicationException(LOGGER, e.getMessage(), Response.Status.BAD_REQUEST); } }
@Deprecated @PUT @Path("/tables/{tableName}/segmentConfigs") @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "Update segments configuration", notes = "Updates segmentsConfig section (validation and retention) of a table") @ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Table not found"), @ApiResponse(code = 500, message = "Internal server error")}) public SuccessResponse put(@ApiParam(value = "Table name", required = true) @PathParam("tableName") String tableName, String requestBody) { try { TableConfig tableConfig = TableConfig.fromJsonString(requestBody); pinotHelixResourceManager .updateSegmentsValidationAndRetentionConfigFor(tableConfig.getTableName(), tableConfig.getTableType(), tableConfig.getValidationConfig()); return new SuccessResponse("Update segmentsConfig for table: " + tableName); } catch (IOException e) { metrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_TABLE_SCHEMA_UPDATE_ERROR, 1L); throw new ControllerApplicationException(LOGGER, String.format("Invalid json while updating segments config for table: %s", tableName), Response.Status.BAD_REQUEST, e); } catch (Exception e) { metrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_TABLE_SCHEMA_UPDATE_ERROR, 1L); throw new ControllerApplicationException(LOGGER, String.format("Failed to update segments config for table: %s", tableName), Response.Status.INTERNAL_SERVER_ERROR, e); } } }
/** * Rebalances ideal state object without updating it * @param idealState * @param tableConfig * @param targetNumReplicas * @param rebalanceUserConfig * @param newPartitionAssignment */ private IdealState rebalanceIdealState(IdealState idealState, TableConfig tableConfig, int targetNumReplicas, Configuration rebalanceUserConfig, PartitionAssignment newPartitionAssignment) { // if realtime and includeConsuming, then rebalance consuming segments if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { boolean includeConsuming = rebalanceUserConfig.getBoolean(RebalanceUserConfigConstants.INCLUDE_CONSUMING, DEFAULT_INCLUDE_CONSUMING); if (includeConsuming) { rebalanceConsumingSegments(idealState, newPartitionAssignment); } } // always rebalance serving segments rebalanceServingSegments(idealState, tableConfig, targetNumReplicas); idealState.setReplicas(Integer.toString(targetNumReplicas)); return idealState; }
throws InvalidConfigException { if (tableConfig.getTableType() == CommonConstants.Helix.TableType.REALTIME) { throw new InvalidConfigException("Realtime table is not supported by replica group rebalancer");
private void getServingInstances(TableConfig tableConfig, List<String> servingInstances, List<String> enabledServingInstances) { String tag; if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(tableConfig); tag = realtimeTagConfig.getCompletedServerTag(); } else { OfflineTagConfig offlineTagConfig = new OfflineTagConfig(tableConfig); tag = offlineTagConfig.getOfflineServerTag(); } servingInstances.addAll(getInstancesWithTag(tag)); enabledServingInstances.addAll(getEnabledInstancesWithTag(tag)); }
if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { filterSegmentsForRealtimeRebalance(mapFields, removedEntries);
if (tableConfig.getTableType() != CommonConstants.Helix.TableType.OFFLINE) { LOGGER.warn("Skip generating ConvertToRawIndexTask for non-OFFLINE table: {}", offlineTableName); continue;
/** * If realtime table and includeConsuming=true, rebalance consuming segments. NewPartitionAssignment will be used only * in this case. Always rebalance completed (online) segments, NewPartitionAssignment unused in this case * @param idealState old ideal state * @param tableConfig table config of table tor rebalance * @param rebalanceUserConfig custom user configs for specific rebalance strategies * @param newPartitionAssignment new rebalanced partition assignments as part of the resource rebalance * @return */ @Override public IdealState getRebalancedIdealState(IdealState idealState, TableConfig tableConfig, Configuration rebalanceUserConfig, PartitionAssignment newPartitionAssignment) { String tableNameWithType = tableConfig.getTableName(); CommonConstants.Helix.TableType tableType = tableConfig.getTableType(); LOGGER.info("Rebalancing ideal state for table {}", tableNameWithType); // get target num replicas int targetNumReplicas; if (tableType.equals(CommonConstants.Helix.TableType.REALTIME)) { String replicasString = tableConfig.getValidationConfig().getReplicasPerPartition(); try { targetNumReplicas = Integer.parseInt(replicasString); } catch (Exception e) { throw new RuntimeException("Invalid value for replicasPerPartition:'" + replicasString + "'", e); } } else { targetNumReplicas = Integer.parseInt(tableConfig.getValidationConfig().getReplication()); } return rebalanceIdealState(idealState, tableConfig, targetNumReplicas, rebalanceUserConfig, newPartitionAssignment); }
@Nonnull @Override public List<PinotTaskConfig> generateTasks(@Nonnull List<TableConfig> tableConfigs) { assertEquals(tableConfigs.size(), 2); // Generate at most 2 tasks if (_clusterInfoProvider.getTaskStates(TASK_TYPE).size() >= 2) { return Collections.emptyList(); } List<PinotTaskConfig> taskConfigs = new ArrayList<>(); for (TableConfig tableConfig : tableConfigs) { Map<String, String> configs = new HashMap<>(); configs.put("tableName", tableConfig.getTableName()); configs.put("tableType", tableConfig.getTableType().toString()); taskConfigs.add(new PinotTaskConfig(TASK_TYPE, configs)); } return taskConfigs; }
PartitionAssignment newPartitionAssignment = new PartitionAssignment(tableNameWithType); if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); if (!streamConfig.hasLowLevelConsumerType()) { } else { LOGGER.info("includeConsuming = false. No need to rebalance partition assignment for {}", tableConfig.getTableType());
if (newTableConfig.getTableType() == CommonConstants.Helix.TableType.REALTIME) { if (_pinotHelixResourceManager.hasOfflineTable(rawTableName)) { tableConfigToCompare = _pinotHelixResourceManager.getOfflineTableConfig(rawTableName);
when(mockTableConfig.getValidationConfig()).thenReturn(mockValidationConfig); CommonConstants.Helix.TableType tableTypeFromTableName = TableNameBuilder.getTableTypeFromTableName(tableName); when(mockTableConfig.getTableType()).thenReturn(tableTypeFromTableName);
CommonConstants.Helix.TableType tableType = tableConfig.getTableType(); if (tableType == CommonConstants.Helix.TableType.OFFLINE) { offlineQuotaConfig = tableConfig.getQuotaConfig();
CommonConstants.Helix.TableType tableType = tableConfig.getTableType(); String configTableName = tableConfig.getTableName(); String tableNameWithType = TableNameBuilder.forType(tableType).tableNameWithType(tableName);
boolean verifyReplication = true; if (config.getTableType() == CommonConstants.Helix.TableType.REALTIME) { StreamConfig streamConfig; try {
/** * * @param tableConfig * @param emptyIdealState may contain HLC segments if both HLC and LLC are configured */ public void setupNewTable(TableConfig tableConfig, IdealState emptyIdealState) throws InvalidConfigException { final StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); int partitionCount = getPartitionCount(streamConfig); List<String> currentSegments = getExistingSegments(tableConfig.getTableName()); // Make sure that there are no low-level segments existing. if (currentSegments != null) { for (String segment : currentSegments) { if (!SegmentName.isHighLevelConsumerSegmentName(segment)) { // For now, we don't support re-creating the low-level realtime segments throw new RuntimeException("Low-level segments already exist for table " + tableConfig.getTableType()); } } } _flushThresholdUpdateManager.clearFlushThresholdUpdater(tableConfig); if (!isConnected()) { throw new RuntimeException( "Lost zk connection while setting up new table " + tableConfig.getTableName() + " isConnected=" + isConnected()); } IdealState idealState = setupTable(tableConfig, emptyIdealState, partitionCount); setTableIdealState(tableConfig.getTableName(), idealState); }
if (tableConfig.getTableType() == TableType.OFFLINE) { buildNameEnum = RoutingTableBuilderName.DefaultOffline; } else if (tableConfig.getTableType() == TableType.REALTIME) { buildNameEnum = RoutingTableBuilderName.DefaultRealtime; } else {