/** * Factory method for initialize a instance. * @param conf config. * @return an instance of {@link AssignmentDistributionService} */ public static AssignmentDistributionService getInstance(Map conf) { AssignmentDistributionService service = new AssignmentDistributionService(); service.prepare(conf); return service; }
public MockLeaderElector(boolean isLeader, String host, int port) { this.isLeader = isLeader; this.leaderAddress = new NimbusInfo(host, port, true); }
/** * Used for local cluster. * * @param supervisor {@link org.apache.storm.daemon.supervisor.Supervisor} */ public void addSupervisor(org.apache.storm.daemon.supervisor.Supervisor supervisor) { assignmentsDistributer.addLocalSupervisor(supervisor); }
protected ILeaderElector zkLeaderElectorImpl(Map<String, Object> conf, CuratorFramework zk, BlobStore blobStore, final TopoCache tc, IStormClusterState clusterState, List<ACL> acls, StormMetricsRegistry metricsRegistry) throws UnknownHostException { List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); String leaderLockPath = "/leader-lock"; String id = NimbusInfo.fromConf(conf).toHostPortString(); AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id)); AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference = new AtomicReference<>(leaderLatchListenerImpl( new LeaderListenerCallback(conf, zk, leaderLatchAtomicReference.get(), blobStore, tc, clusterState, acls, metricsRegistry))); return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference, leaderLatchListenerAtomicReference, blobStore, tc, clusterState, acls, metricsRegistry); }
public static NimbusInfo toNimbusInfo(Participant participant) { String id = participant.getId(); if (StringUtils.isBlank(id)) { throw new RuntimeException("No nimbus leader participant host found, have you started your nimbus hosts?"); } NimbusInfo nimbusInfo = NimbusInfo.parse(id); nimbusInfo.setLeader(participant.isLeader()); return nimbusInfo; }
private boolean checkIfStateContainsCurrentNimbusHost(List<String> stateInfoList, NimbusInfo nimbusInfo) { boolean containsNimbusHost = false; for (String stateInfo : stateInfoList) { if (stateInfo.contains(nimbusInfo.getHost())) { containsNimbusHost = true; break; } } return containsNimbusHost; }
@Override public void run() { while (service.isActive()) { try { NodeAssignments nodeAssignments = this.service.nextAssignments(queueIndex); sendAssignmentsToNode(nodeAssignments); } catch (InterruptedException e) { if (service.isActive()) { LOG.error("Get an unexpected interrupt when distributing assignments to node, {}", e.getCause()); } else { // service is off now just interrupt it. Thread.currentThread().interrupt(); } } } }
@Override public void setupBlob(String key, NimbusInfo nimbusInfo, Integer versionInfo) { String path = ClusterUtils.blobstorePath(key) + ClusterUtils.ZK_SEPERATOR + nimbusInfo.toHostPortString() + "-" + versionInfo; LOG.info("set-path: {}", path); stateStorage.mkdirs(ClusterUtils.blobstorePath(key), defaultAcls); stateStorage.delete_node_blobstore(ClusterUtils.blobstorePath(key), nimbusInfo.toHostPortString()); stateStorage.set_ephemeral_node(path, null, defaultAcls); }
private void setUpNimbusInfo(List<ACL> acls) { String leaderInfoPath = conf.get(Config.STORM_ZOOKEEPER_ROOT) + ClusterUtils.LEADERINFO_SUBTREE; NimbusInfo nimbusInfo = NimbusInfo.fromConf(conf); if (ClientZookeeper.existsNode(zk, leaderInfoPath, false)) { ClientZookeeper.setData(zk, leaderInfoPath, Utils.javaSerialize(nimbusInfo)); } else { ClientZookeeper.createNode(zk, leaderInfoPath, Utils.javaSerialize(nimbusInfo), CreateMode.PERSISTENT, acls); } }
/** * Get instance of {@link IWorkerHeartbeatsRecoveryStrategy} with conf. * @param conf strategy config * @return an instance of {@link IWorkerHeartbeatsRecoveryStrategy} */ public static IWorkerHeartbeatsRecoveryStrategy getStrategy(Map<String, Object> conf) { IWorkerHeartbeatsRecoveryStrategy strategy; if (conf.get(DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS) != null) { Object targetObj = ReflectionUtils.newInstance((String) conf.get(DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS)); Preconditions.checkState(targetObj instanceof IWorkerHeartbeatsRecoveryStrategy, "{} must implements IWorkerHeartbeatsRecoveryStrategy", DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS); strategy = ((IWorkerHeartbeatsRecoveryStrategy) targetObj); } else { strategy = new TimeOutWorkerHeartbeatsRecoveryStrategy(); } strategy.prepare(conf); return strategy; }
public static NodeAssignments getInstance(String node, String host, Integer serverPort, SupervisorAssignments assignments) { return new NodeAssignments(node, host, serverPort, assignments); }
public static void main(String[] args) { testMaxThroughput(); }
private boolean isLeader() throws Exception { return leaderElector.isLeader(); }
@Override public void notLeader() { LOG.info("{} lost leadership.", hostName); //Just to be sure callback.notLeaderCallback(); } };
private void notifyTopologyActionListener(String topoId, String action) { ITopologyActionNotifierPlugin notifier = nimbusTopologyActionNotifier; if (notifier != null) { try { notifier.notify(topoId, action); } catch (Exception e) { LOG.warn("Ignoring exception from Topology action notifier for storm-Id {}", topoId, e); } } }
@Override public void isLeader() { callback.leaderCallBack(); LOG.info("{} gained leadership.", hostName); }
private LinkedBlockingQueue<NodeAssignments> nextQueue() { return this.assignmentsQueue.get(nextQueueId()); }
private void assertIsLeader() throws Exception { if (!isLeader()) { NimbusInfo leaderAddress = leaderElector.getLeader(); throw new RuntimeException("not a leader, current leader is " + leaderAddress); } }
private static SupervisorWorkerHeartbeats mockedHeartbeats() { SupervisorWorkerHeartbeats heartbeats = new SupervisorWorkerHeartbeats(); heartbeats.set_supervisor_id("123124134123413412341351234143"); List<SupervisorWorkerHeartbeat> workers = new ArrayList<>(); for (int i = 0; i < 25; i++) { workers.add(nextMockedWorkerbeat()); } heartbeats.set_worker_heartbeats(workers); return heartbeats; }
@Override public void notify(TopologyInfo topologyInfo, Map<String, Object> topoConf, StormTopology topology) throws IllegalAccessException { submittedTopologies.add(new TopologyDetails(topologyInfo.get_name(), topology)); } }