public TagConfig(TableConfig tableConfig) { _tableConfig = tableConfig; _serverTenant = tableConfig.getTenantConfig().getServer(); }
private String getServerTenantName(String tableName) throws IOException { return getTableConfig(tableName).getTenantConfig().getServer(); }
private String getTablesServedFromTenant(String tenantName) { Set<String> tables = new HashSet<>(); ObjectNode resourceGetRet = JsonUtils.newObjectNode(); for (String table : pinotHelixResourceManager.getAllTables()) { TableConfig tableConfig = pinotHelixResourceManager.getTableConfig(table); String tableConfigTenant = tableConfig.getTenantConfig().getServer(); if (tenantName.equals(tableConfigTenant)) { tables.add(table); } } resourceGetRet.set(TABLES, JsonUtils.objectToJsonNode(tables)); return resourceGetRet.toString(); }
public List<String> getAllTenantTables() throws Exception { String tableConfigPath = "/CONFIGS/TABLE"; List<ZNRecord> tableConfigs = _propertyStore.getChildren(tableConfigPath, null, 0); List<String> tables = new ArrayList<>(128); for (ZNRecord znRecord : tableConfigs) { TableConfig tableConfig = TableConfig.fromZnRecord(znRecord); if (tableConfig.getTenantConfig().getServer().equals(_tenantName)) { tables.add(tableConfig.getTableName()); } } return tables; }
@Override protected void processTable(String tableNameWithType) { TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType); if (tableConfig == null) { LOGGER.warn("Failed to find table config for table: {}, skipping broker resource validation", tableNameWithType); return; } // Rebuild broker resource Set<String> brokerInstances = _pinotHelixResourceManager .getAllInstancesForBrokerTenant(_instanceConfigs, tableConfig.getTenantConfig().getBroker()); _pinotHelixResourceManager.rebuildBrokerResource(tableNameWithType, brokerInstances); }
List<String> getAllRealtimeTablesWithServerTenant(String serverTenant) { List<String> realtimeTablesWithServerTenant = new ArrayList<>(); for (Map.Entry<String, TableConfig> entry : _tableConfigsStore.entrySet()) { if (entry.getValue().getTenantConfig().getServer().equals(serverTenant)) { realtimeTablesWithServerTenant.add(entry.getKey()); } } return realtimeTablesWithServerTenant; } }
/** * Get all the broker instances for the given table name. * * @param tableName Table name with or without type suffix * @return List of broker instance Ids */ @Nonnull public List<String> getBrokerInstancesFor(@Nonnull String tableName) { String brokerTenantName = null; TableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName); if (offlineTableConfig != null) { brokerTenantName = offlineTableConfig.getTenantConfig().getBroker(); } else { TableConfig realtimeTableConfig = ZKMetadataProvider.getRealtimeTableConfig(_propertyStore, tableName); if (realtimeTableConfig != null) { brokerTenantName = realtimeTableConfig.getTenantConfig().getBroker(); } } return HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getBrokerTagForTenant(brokerTenantName)); }
/** * Rebalances a table * @param tableName * @throws Exception */ public void rebalanceTable(String tableName) throws Exception { String tableConfigPath = "/CONFIGS/TABLE/" + tableName; Stat stat = new Stat(); ZNRecord znRecord = propertyStore.get(tableConfigPath, stat, 0); TableConfig tableConfig = TableConfig.fromZnRecord(znRecord); String tenantName = tableConfig.getTenantConfig().getServer().replaceAll(TableType.OFFLINE.toString(), "") .replace(TableType.OFFLINE.toString(), ""); rebalanceTable(tableName, tenantName); }
/** * rebalances all tables for the tenant * @param tenantName */ public void rebalanceTenantTables(String tenantName) throws Exception { String tableConfigPath = "/CONFIGS/TABLE"; List<Stat> stats = new ArrayList<>(); List<ZNRecord> tableConfigs = propertyStore.getChildren(tableConfigPath, stats, 0); String rawTenantName = tenantName.replaceAll("_OFFLINE", "").replace("_REALTIME", ""); int nRebalances = 0; for (ZNRecord znRecord : tableConfigs) { TableConfig tableConfig; try { tableConfig = TableConfig.fromZnRecord(znRecord); } catch (Exception e) { LOGGER.warn("Failed to parse table configuration for ZnRecord id: {}. Skipping", znRecord.getId()); continue; } if (tableConfig.getTenantConfig().getServer().equals(rawTenantName)) { LOGGER.info(tableConfig.getTableName() + ":" + tableConfig.getTenantConfig().getServer()); nRebalances++; rebalanceTable(tableConfig.getTableName(), tenantName); } } if (nRebalances == 0) { LOGGER.info("No tables found for tenant " + tenantName); } }
public PinotResourceManagerResponse rebuildBrokerResourceFromHelixTags(String tableNameWithType) throws Exception { TableConfig tableConfig; try { tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); } catch (Exception e) { LOGGER.warn("Caught exception while getting table config for table {}", tableNameWithType, e); throw new InvalidTableConfigException( "Failed to fetch broker tag for table " + tableNameWithType + " due to exception: " + e.getMessage()); } if (tableConfig == null) { LOGGER.warn("Table " + tableNameWithType + " does not exist"); throw new InvalidConfigException( "Invalid table configuration for table " + tableNameWithType + ". Table does not exist"); } return rebuildBrokerResource(tableNameWithType, getAllInstancesForBrokerTenant(tableConfig.getTenantConfig().getBroker())); }
public List<String> getBrokerInstancesForTable(String tableName, TableType tableType) { TableConfig tableConfig = getTableConfig(tableName, tableType); String brokerTenantName = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); List<String> serverInstances = HelixHelper.getInstancesWithTag(_helixZkManager, brokerTenantName); return serverInstances; }
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) { IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); for (String tableNameWithType : tableIdealState.getPartitionSet()) { TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); String brokerTag = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); if (brokerTag.equals(brokerTenantTag)) { tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerOnlineOfflineStateModel.ONLINE); } } _helixAdmin .setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState); }
private void validateLoadedConfig(TableConfig config) { Assert.assertEquals(config.getTableName(), "mytable_OFFLINE"); Assert.assertEquals(config.getQuotaConfig().getStorage(), "125 GiB"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeValue(), "5"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeUnit(), "DAYS"); Assert.assertEquals(config.getTenantConfig().getBroker(), "foo"); Assert.assertEquals(config.getTenantConfig().getServer(), "bar"); Assert.assertEquals(config.getIndexingConfig().getSortedColumn(), Lists.newArrayList("foo")); } }
public RealtimeTagConfig(TableConfig tableConfig) { super(tableConfig); TagOverrideConfig tagOverrideConfig = tableConfig.getTenantConfig().getTagOverrideConfig(); if (tagOverrideConfig != null && tagOverrideConfig.getRealtimeConsuming() != null) { _consumingRealtimeServerTag = tagOverrideConfig.getRealtimeConsuming(); } else { _consumingRealtimeServerTag = TagNameUtils.getRealtimeTagForTenant(_serverTenant); } if (tagOverrideConfig != null && tagOverrideConfig.getRealtimeCompleted() != null) { _completedRealtimeServerTag = tagOverrideConfig.getRealtimeCompleted(); } else { _completedRealtimeServerTag = TagNameUtils.getRealtimeTagForTenant(_serverTenant); } if (!_consumingRealtimeServerTag.equals(_completedRealtimeServerTag)) { _relocateCompletedSegments = true; } }
TenantConfig mockTenantConfig = mock(TenantConfig.class); when(mockTenantConfig.getServer()).thenReturn(serverTenant); when(mockTableConfig.getTenantConfig()).thenReturn(mockTenantConfig);
/** * Helper method to add the passed in offline segment to the helix cluster. * - Gets the segment name and the table name from the passed in segment meta-data. * - Identifies the instance set onto which the segment needs to be added, based on * segment assignment strategy and replicas in the table config in the property-store. * - Updates ideal state such that the new segment is assigned to required set of instances as per * the segment assignment strategy and replicas. * * @param segmentMetadata Meta-data for the segment, used to access segmentName and tableName. */ // NOTE: method should be thread-safe private void addNewOfflineSegment(SegmentMetadata segmentMetadata) { String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(segmentMetadata.getTableName()); String segmentName = segmentMetadata.getName(); // Assign new segment to instances TableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, offlineTableName); Preconditions.checkNotNull(offlineTableConfig); int numReplicas = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication()); String serverTenant = TagNameUtils.getOfflineTagForTenant(offlineTableConfig.getTenantConfig().getServer()); SegmentAssignmentStrategy segmentAssignmentStrategy = SegmentAssignmentStrategyFactory .getSegmentAssignmentStrategy(offlineTableConfig.getValidationConfig().getSegmentAssignmentStrategy()); List<String> assignedInstances = segmentAssignmentStrategy .getAssignedInstances(_helixZkManager, _helixAdmin, _propertyStore, _helixClusterName, segmentMetadata, numReplicas, serverTenant); HelixHelper.addSegmentToIdealState(_helixZkManager, offlineTableName, segmentName, assignedInstances); }
TenantConfig mockTenantConfig = mock((TenantConfig.class)); when(mockTenantConfig.getServer()).thenReturn(aServerTenant); when(tableConfig.getTenantConfig()).thenReturn(mockTenantConfig); SegmentsValidationAndRetentionConfig mockValidationConfig = mock(SegmentsValidationAndRetentionConfig.class); when(mockValidationConfig.getReplicasPerPartitionNumber()).thenReturn(2);
Assert.assertEquals(tableConfig.getTableType(), TableType.OFFLINE); Assert.assertEquals(tableConfig.getIndexingConfig().getLoadMode(), "HEAP"); Assert.assertNotNull(tableConfig.getTenantConfig()); Assert.assertEquals(tableConfig.getTenantConfig().getServer(), "aServerTenant"); Assert.assertEquals(tableConfig.getTenantConfig().getBroker(), "aBrokerTenant"); Assert.assertNull(tableConfig.getTenantConfig().getTagOverrideConfig()); Assert.assertNotNull(tableConfigToCompare.getTenantConfig()); Assert .assertEquals(tableConfigToCompare.getTenantConfig().getServer(), tableConfig.getTenantConfig().getServer()); Assert .assertEquals(tableConfigToCompare.getTenantConfig().getBroker(), tableConfig.getTenantConfig().getBroker()); Assert.assertNull(tableConfig.getTenantConfig().getTagOverrideConfig()); Assert.assertNotNull(tableConfigToCompare.getTenantConfig()); Assert .assertEquals(tableConfigToCompare.getTenantConfig().getServer(), tableConfig.getTenantConfig().getServer()); Assert .assertEquals(tableConfigToCompare.getTenantConfig().getBroker(), tableConfig.getTenantConfig().getBroker()); Assert.assertNull(tableConfig.getTenantConfig().getTagOverrideConfig()); Assert.assertNotNull(tableConfig.getTenantConfig()); Assert.assertEquals(tableConfig.getTenantConfig().getServer(), "aServerTenant"); Assert.assertEquals(tableConfig.getTenantConfig().getBroker(), "aBrokerTenant"); Assert.assertNotNull(tableConfig.getTenantConfig().getTagOverrideConfig()); Assert.assertEquals(tableConfig.getTenantConfig().getTagOverrideConfig().getRealtimeConsuming(), "aRTConsumingTag_REALTIME"); Assert.assertNull(tableConfig.getTenantConfig().getTagOverrideConfig().getRealtimeCompleted());
tableConfig.setTenantConfig(tenantConfig); } else { tenantConfig = tableConfig.getTenantConfig(); if (tenantConfig.getBroker() == null || tenantConfig.getServer() == null) { throw new InvalidTableConfigException("Tenant is not configured for table: " + tableNameWithType);