/** * Returns the broker instances in the cluster for the given tenant. * * TODO: refactor code to use this method if applicable to reuse instance configs in order to reduce ZK accesses */ public static Set<String> getBrokerInstancesForTenant(List<InstanceConfig> instanceConfigs, String tenant) { return new HashSet<>(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getBrokerTagForTenant(tenant))); } }
public static String getTagFromTenantAndServerType(String tenantName, ServerType type) { if (type == ServerType.OFFLINE) { return getOfflineTagForTenant(tenantName); } return getRealtimeTagForTenant(tenantName); }
public static String getBrokerTagForTenant(String tenantName) { if (tenantName == null) { return TagNameUtils.getBrokerTagForTenant(DEFAULT_TENANT_NAME); } if (tenantName.endsWith(TenantRole.BROKER.toString())) { return tenantName; } else { return TagNameUtils.buildBrokerTenantTagFromTenantName(tenantName); } }
public static String getRealtimeTagForTenant(String tenantName) { if (tenantName == null) { return TagNameUtils.getRealtimeTagForTenant(DEFAULT_TENANT_NAME); } if (tenantName.endsWith(ServerType.REALTIME.toString())) { return tenantName; } else { return TagNameUtils.buildRealtimeTagFromTenantName(tenantName); } }
public boolean isTenantExisted(String tenantName) { if (!HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getBrokerTagForTenant(tenantName)).isEmpty()) { return true; } if (!HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getOfflineTagForTenant(tenantName)).isEmpty()) { return true; } if (!HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getRealtimeTagForTenant(tenantName)).isEmpty()) { return true; } return false; }
public OfflineTagConfig(TableConfig tableConfig) { super(tableConfig); _offlineServerTag = TagNameUtils.getOfflineTagForTenant(_serverTenant); }
String brokerTenantName = TagNameUtils.getBrokerTagForTenant(tenantConfig.getBroker()); List<String> brokersForTenant = HelixHelper.getInstancesWithTag(_helixZkManager, brokerTenantName); if (brokersForTenant.isEmpty()) { TagNameUtils.getTagFromTenantAndServerType(tenantConfig.getServer(), tableType.getServerType()); if (HelixHelper.getInstancesWithTag(_helixZkManager, serverTenantName).isEmpty()) { throw new InvalidTableConfigException( String realtimeConsumingTag = tagOverrideConfig.getRealtimeConsuming(); if (realtimeConsumingTag != null) { if (!TagNameUtils.hasValidServerTagSuffix(realtimeConsumingTag)) { throw new InvalidTableConfigException( "Invalid realtime consuming tag: " + realtimeConsumingTag + ". Must have suffix _REALTIME or _OFFLINE"); if (!TagNameUtils.hasValidServerTagSuffix(realtimeCompletedTag)) { throw new InvalidTableConfigException( "Invalid realtime completed tag: " + realtimeCompletedTag + ". Must have suffix _REALTIME or _OFFLINE");
public static String getOfflineTagForTenant(String tenantName) { if (tenantName == null) { return TagNameUtils.getOfflineTagForTenant(DEFAULT_TENANT_NAME); } if (tenantName.endsWith(ServerType.OFFLINE.toString())) { return tenantName; } else { return TagNameUtils.buildOfflineTagFromTenantName(tenantName); } }
public PinotResourceManagerResponse deleteRealtimeServerTenantFor(String tenantName) { String realtimeTenantTag = TagNameUtils.getRealtimeTagForTenant(tenantName); List<String> instancesInClusterWithTag = HelixHelper.getInstancesWithTag(_helixZkManager, realtimeTenantTag); for (String instanceName : instancesInClusterWithTag) { _helixAdmin.removeInstanceTag(_helixClusterName, instanceName, realtimeTenantTag); if (getTagsForInstance(instanceName).isEmpty()) { _helixAdmin.addInstanceTag(_helixClusterName, instanceName, CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE); } } return PinotResourceManagerResponse.SUCCESS; }
TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getRealtimeTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getOfflineTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20);
@Override public List<String> getAssignedInstances(HelixManager helixManager, HelixAdmin helixAdmin, ZkHelixPropertyStore<ZNRecord> propertyStore, String helixClusterName, SegmentMetadata segmentMetadata, int numReplicas, String tenantName) { String serverTenantName = TagNameUtils.getOfflineTagForTenant(tenantName); final Random random = new Random(System.currentTimeMillis()); List<String> allInstanceList = HelixHelper.getEnabledInstancesWithTag(helixManager, serverTenantName); List<String> selectedInstanceList = new ArrayList<>(); for (int i = 0; i < numReplicas; ++i) { final int idx = random.nextInt(allInstanceList.size()); selectedInstanceList.add(allInstanceList.get(idx)); allInstanceList.remove(idx); } LOGGER.info("Segment assignment result for : " + segmentMetadata.getName() + ", in resource : " + segmentMetadata .getTableName() + ", selected instances: " + Arrays.toString(selectedInstanceList.toArray())); return selectedInstanceList; } }
public RealtimeTagConfig(TableConfig tableConfig) { super(tableConfig); TagOverrideConfig tagOverrideConfig = tableConfig.getTenantConfig().getTagOverrideConfig(); if (tagOverrideConfig != null && tagOverrideConfig.getRealtimeConsuming() != null) { _consumingRealtimeServerTag = tagOverrideConfig.getRealtimeConsuming(); } else { _consumingRealtimeServerTag = TagNameUtils.getRealtimeTagForTenant(_serverTenant); } if (tagOverrideConfig != null && tagOverrideConfig.getRealtimeCompleted() != null) { _completedRealtimeServerTag = tagOverrideConfig.getRealtimeCompleted(); } else { _completedRealtimeServerTag = TagNameUtils.getRealtimeTagForTenant(_serverTenant); } if (!_consumingRealtimeServerTag.equals(_completedRealtimeServerTag)) { _relocateCompletedSegments = true; } }
/** * Returns the server instances in the cluster for the given tenant. * * TODO: refactor code to use this method if applicable to reuse instance configs in order to reduce ZK accesses */ public static Set<String> getServerInstancesForTenant(List<InstanceConfig> instanceConfigs, String tenant) { Set<String> serverInstances = new HashSet<>(); serverInstances .addAll(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getOfflineTagForTenant(tenant))); serverInstances .addAll(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getRealtimeTagForTenant(tenant))); return serverInstances; }
public PinotResourceManagerResponse deleteBrokerTenantFor(String tenantName) { String brokerTag = TagNameUtils.getBrokerTagForTenant(tenantName); List<String> instancesInClusterWithTag = HelixHelper.getInstancesWithTag(_helixZkManager, brokerTag); for (String instance : instancesInClusterWithTag) { retagInstance(instance, brokerTag, CommonConstants.Helix.UNTAGGED_BROKER_INSTANCE); } return PinotResourceManagerResponse.SUCCESS; }
public PinotResourceManagerResponse deleteOfflineServerTenantFor(String tenantName) { String offlineTenantTag = TagNameUtils.getOfflineTagForTenant(tenantName); List<String> instancesInClusterWithTag = HelixHelper.getInstancesWithTag(_helixZkManager, offlineTenantTag); for (String instanceName : instancesInClusterWithTag) { _helixAdmin.removeInstanceTag(_helixClusterName, instanceName, offlineTenantTag); if (getTagsForInstance(instanceName).isEmpty()) { _helixAdmin.addInstanceTag(_helixClusterName, instanceName, CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE); } } return PinotResourceManagerResponse.SUCCESS; }
int nConsumingInstances = 3; String serverTenant = "aServerTenant"; String realtimeTagForTenant = TagNameUtils.getRealtimeTagForTenant(serverTenant); PartitionAssignment newPartitionAssignment = new PartitionAssignment(realtimeTableName); List<String> completedInstances = getInstanceList(nCompletedInstances);
private void assignColocatedServerTenant(Tenant serverTenant, int numberOfInstances, List<String> unTaggedInstanceList) { int cnt = 0; String offlineServerTag = TagNameUtils.getOfflineTagForTenant(serverTenant.getTenantName()); for (int i = 0; i < serverTenant.getOfflineInstances(); i++) { retagInstance(unTaggedInstanceList.get(cnt++), CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE, offlineServerTag); } String realtimeServerTag = TagNameUtils.getRealtimeTagForTenant(serverTenant.getTenantName()); for (int i = 0; i < serverTenant.getRealtimeInstances(); i++) { retagInstance(unTaggedInstanceList.get(cnt++), CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE, realtimeServerTag); if (cnt == numberOfInstances) { cnt = 0; } } }
public boolean isBrokerTenantDeletable(String tenantName) { String brokerTag = TagNameUtils.getBrokerTagForTenant(tenantName); Set<String> taggedInstances = new HashSet<>(HelixHelper.getInstancesWithTag(_helixZkManager, brokerTag)); String brokerName = CommonConstants.Helix.BROKER_RESOURCE_INSTANCE; IdealState brokerIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, brokerName); for (String partition : brokerIdealState.getPartitionSet()) { for (String instance : brokerIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } return true; }
@Override public List<String> getAssignedInstances(HelixManager helixManager, HelixAdmin helixAdmin, ZkHelixPropertyStore<ZNRecord> propertyStore, String helixClusterName, SegmentMetadata segmentMetadata, int numReplicas, String tenantName) { String serverTenantName = TagNameUtils.getOfflineTagForTenant(tenantName); List<String> allInstances = HelixHelper.getEnabledInstancesWithTag(helixManager, serverTenantName); List<String> selectedInstanceList = new ArrayList<>(); if (segmentMetadata.getShardingKey() != null) { for (String instance : allInstances) { if (HelixHelper.getInstanceConfigsMapFor(instance, helixClusterName, helixAdmin).get("shardingKey") .equalsIgnoreCase(segmentMetadata.getShardingKey())) { selectedInstanceList.add(instance); } } LOGGER.info("Segment assignment result for : " + segmentMetadata.getName() + ", in resource : " + segmentMetadata .getTableName() + ", selected instances: " + Arrays.toString(selectedInstanceList.toArray())); return selectedInstanceList; } else { throw new RuntimeException("Segment missing sharding key!"); } } }
private void assignIndependentServerTenant(Tenant serverTenant, int numberOfInstances, List<String> unTaggedInstanceList) { String offlineServerTag = TagNameUtils.getOfflineTagForTenant(serverTenant.getTenantName()); for (int i = 0; i < serverTenant.getOfflineInstances(); i++) { retagInstance(unTaggedInstanceList.get(i), CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE, offlineServerTag); } String realtimeServerTag = TagNameUtils.getRealtimeTagForTenant(serverTenant.getTenantName()); for (int i = 0; i < serverTenant.getRealtimeInstances(); i++) { retagInstance(unTaggedInstanceList.get(i + serverTenant.getOfflineInstances()), CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE, realtimeServerTag); } }