public BrokerResourceOnlineOfflineStateModelFactory(HelixManager helixManager, ZkHelixPropertyStore<ZNRecord> propertyStore, HelixExternalViewBasedRouting helixExternalViewBasedRouting, TableQueryQuotaManager tableQueryQuotaManager) { _helixManager = helixManager; _propertyStore = propertyStore; _helixAdmin = helixManager.getClusterManagmentTool(); _helixExternalViewBasedRouting = helixExternalViewBasedRouting; _tableQueryQuotaManager = tableQueryQuotaManager; }
public IdealStateMatchServiceStatusCallback(HelixManager helixManager, String clusterName, String instanceName, List<String> resourcesToMonitor) { _clusterName = clusterName; _instanceName = instanceName; _helixAdmin = helixManager.getClusterManagmentTool(); _helixDataAccessor = helixManager.getHelixDataAccessor(); _resourcesToMonitor = new HashSet<>(resourcesToMonitor); _numTotalResourcesToMonitor = _resourcesToMonitor.size(); LOGGER.info("Monitoring {} resources: {} for start up of instance {}", _numTotalResourcesToMonitor, _resourcesToMonitor, _instanceName); }
public ReplicaGroupRebalanceSegmentStrategy(HelixManager helixManager) { _helixManager = helixManager; _helixAdmin = helixManager.getClusterManagmentTool(); _helixClusterName = helixManager.getClusterName(); _propertyStore = helixManager.getHelixPropertyStore(); }
private boolean noOnlineResources(HelixManager spectatorManager, Set<String> resources) { Iterator<String> iterator = resources.iterator(); while (iterator.hasNext()) { String resourceName = iterator.next(); ExternalView externalView = spectatorManager.getClusterManagmentTool().getResourceExternalView(_helixClusterName, resourceName); if (externalView == null) { iterator.remove(); continue; } for (String partition : externalView.getPartitionSet()) { Map<String, String> instanceStateMap = externalView.getStateMap(partition); if (instanceStateMap.containsKey(_instanceId)) { if ("ONLINE".equals(instanceStateMap.get(_instanceId))) { return false; } } } iterator.remove(); } return true; }
protected ExternalView fetchExternalView(String table) { return HelixHelper .getExternalViewForResource(_helixManager.getClusterManagmentTool(), _helixManager.getClusterName(), table); }
public DefaultRebalanceSegmentStrategy(HelixManager helixManager) { _helixManager = helixManager; _helixAdmin = helixManager.getClusterManagmentTool(); _helixClusterName = helixManager.getClusterName(); _propertyStore = helixManager.getHelixPropertyStore(); }
/** * Helix participant cannot pre-configure tags before it connects to ZK. So this method can only be invoked after * {@link HelixManager#connect()}. However this will still work because tagged jobs won't be sent to a non-tagged instance. Hence * the job with EXAMPLE_INSTANCE_TAG will remain in the ZK until an instance with EXAMPLE_INSTANCE_TAG was found. */ private void addInstanceTags() { List<String> tags = ConfigUtils.getStringList(this.config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_KEY); HelixManager receiverManager = getReceiverManager(); if (receiverManager.isConnected()) { if (!tags.isEmpty()) { logger.info("Adding tags binding " + tags); tags.forEach(tag -> receiverManager.getClusterManagmentTool() .addInstanceTag(this.clusterName, this.helixInstanceName, tag)); logger.info("Actual tags binding " + receiverManager.getClusterManagmentTool() .getInstanceConfig(this.clusterName, this.helixInstanceName).getTags()); } } }
spectatorManager.connect(); Set<String> resources = fetchLatestTableResources(spectatorManager.getClusterManagmentTool());
public IdealStateMatchServiceStatusCallback(HelixManager helixManager, String clusterName, String instanceName) { _clusterName = clusterName; _instanceName = instanceName; _helixAdmin = helixManager.getClusterManagmentTool(); _helixDataAccessor = helixManager.getHelixDataAccessor(); _resourcesToMonitor = new HashSet<>(); for (String resourceName : _helixAdmin.getResourcesInCluster(_clusterName)) { // Only monitor table resources and broker resource if (!TableNameBuilder.isTableResource(resourceName) && !resourceName .equals(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) { continue; } // Only monitor enabled resources IdealState idealState = getResourceIdealState(resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceName)) { _resourcesToMonitor.add(resourceName); break; } } } } _numTotalResourcesToMonitor = _resourcesToMonitor.size(); LOGGER.info("Monitoring {} resources: {} for start up of instance {}", _numTotalResourcesToMonitor, _resourcesToMonitor, _instanceName); }
public static void addFakeDataInstanceToAutoJoinHelixCluster(String helixClusterName, String zkServer, String instanceId, boolean isSingleTenant, int adminPort) throws Exception { final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, instanceId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptySegmentOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptySegmentOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.OFFLINE.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, UNTAGGED_SERVER_INSTANCE); } HelixConfigScope scope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.PARTICIPANT, helixClusterName) .forParticipant(instanceId).build(); Map<String, String> props = new HashMap<>(); props.put(CommonConstants.Helix.Instance.ADMIN_PORT_KEY, String.valueOf(adminPort)); helixZkManager.getClusterManagmentTool().setConfig(scope, props); }
.getExternalViewForResource(_helixManager.getClusterManagmentTool(), _helixManager.getClusterName(), BROKER_RESOURCE_INSTANCE); if (currentBrokerResource == null) {
public static void addFakeBrokerInstancesToAutoJoinHelixCluster(String helixClusterName, String zkServer, int numInstances, boolean isSingleTenant) throws Exception { for (int i = 0; i < numInstances; ++i) { final String brokerId = "Broker_localhost_" + i; final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptyBrokerOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptyBrokerOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, UNTAGGED_BROKER_INSTANCE); } } }
/** * Create Helix cluster if needed, and then start a Pinot controller instance. */ public synchronized void start() { _helixZkManager = HelixSetupUtils .setup(_helixClusterName, _helixZkURL, _instanceId, _isUpdateStateModel, _enableBatchMessageMode); Preconditions.checkNotNull(_helixZkManager); _helixAdmin = _helixZkManager.getClusterManagmentTool(); _propertyStore = _helixZkManager.getHelixPropertyStore(); _helixDataAccessor = _helixZkManager.getHelixDataAccessor(); // Cache instance zk paths. BaseDataAccessor<ZNRecord> baseDataAccessor = _helixDataAccessor.getBaseDataAccessor(); String instanceConfigs = PropertyPathBuilder.instanceConfig(_helixClusterName); _cacheInstanceConfigsDataAccessor = new ZkCacheBaseDataAccessor<>((ZkBaseDataAccessor<ZNRecord>) baseDataAccessor, instanceConfigs, null, Collections.singletonList(instanceConfigs)); _keyBuilder = _helixDataAccessor.keyBuilder(); _segmentDeletionManager = new SegmentDeletionManager(_dataDir, _helixAdmin, _helixClusterName, _propertyStore); ZKMetadataProvider.setClusterTenantIsolationEnabled(_propertyStore, _isSingleTenantCluster); _tableRebalancer = new TableRebalancer(_helixZkManager, _helixAdmin, _helixClusterName); }
new TaskFactoryRegistry(_taskExecutorFactoryRegistry, _eventObserverFactoryRegistry).getTaskFactoryRegistry())); _helixManager.connect(); _helixAdmin = _helixManager.getClusterManagmentTool(); addInstanceTagIfNeeded();
_helixZkManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, helixZkURL, instanceId, /*isUpdateStateModel=*/false, true); _helixAdmin = _helixZkManager.getClusterManagmentTool(); _partitionAssignmentGenerator = new ReplicaGroupPartitionAssignmentGenerator(_helixZkManager.getHelixPropertyStore());
final StateMachineEngine stateMachineEngine = _helixManager.getStateMachineEngine(); _helixManager.connect(); _helixAdmin = _helixManager.getClusterManagmentTool(); addInstanceTagIfNeeded(helixClusterName, _instanceId); ZkHelixPropertyStore<ZNRecord> propertyStore = _helixManager.getHelixPropertyStore();
HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.SPECTATOR, zkServers); _spectatorHelixManager.connect(); _helixAdmin = _spectatorHelixManager.getClusterManagmentTool(); _propertyStore = _spectatorHelixManager.getHelixPropertyStore(); _helixExternalViewBasedRouting = new HelixExternalViewBasedRouting(_propertyStore, _spectatorHelixManager,
HelixAdmin helixAdmin = _helixManager.getClusterManagmentTool();
/** * Helper method to start a fake server that only implements Helix part. * * @throws Exception */ private void startFakeServer() throws Exception { _serverInstance = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + NetUtil.getHostAddress() + "_" + CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT; // Create server instance with the fake server state model _serverHelixManager = HelixManagerFactory .getZKHelixManager(_clusterName, _serverInstance, InstanceType.PARTICIPANT, ZkStarter.DEFAULT_ZK_STR); _serverHelixManager.getStateMachineEngine() .registerStateModelFactory(SegmentOnlineOfflineStateModelFactory.getStateModelName(), new FakeServerSegmentStateModelFactory()); _serverHelixManager.connect(); // Add Helix tag to the server _serverHelixManager.getClusterManagmentTool().addInstanceTag(_clusterName, _serverInstance, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); // Initialize controller leader locator ControllerLeaderLocator.create(_serverHelixManager); }
private static void assignIdealStates(HelixManager helixManager, Map<String, IdealState> idealStatesFromAssignment) { HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : idealStatesFromAssignment.keySet()) { IdealState idealState = idealStatesFromAssignment.get(topic); helixAdmin.setResourceIdealState(helixClusterName, topic, idealState); } }