public MockLeaderElector(boolean isLeader, String host, int port) { this.isLeader = isLeader; this.leaderAddress = new NimbusInfo(host, port, true); }
public static void main(String[] args) throws Exception { if(args.length != 1) { System.out.println("Need 1 arguments, but have " + Integer.toString(args.length)); System.out.println("listLocalFs <local_blobstore_dir>"); System.out.println("Migrates blobs from LocalFsBlobStore to HdfsBlobStore"); System.out.println("Example: listLocalFs '/srv/storm'"); System.exit(1); } Map<String, Object> lfsConf = Utils.readStormConfig(); lfsConf.put(Config.BLOBSTORE_DIR, args[0]); lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); /* CREATE THE BLOBSTORE */ LocalFsBlobStore lfsBlobStore = new LocalFsBlobStore(); lfsBlobStore.prepare(lfsConf, null, NimbusInfo.fromConf(lfsConf), null); /* LOOK AT HDFS BLOBSTORE */ System.out.println("Listing Local blobstore keys."); MigratorMain.listBlobStoreKeys(lfsBlobStore, null); System.out.println("Done Listing Local blobstore keys."); lfsBlobStore.shutdown(); } }
public static void createStateInZookeeper(Map<String, Object> conf, String key, NimbusInfo nimbusInfo) throws TTransportException { ClientBlobStore cb = new NimbusBlobStore(); cb.setClient(conf, new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null)); cb.createStateInZookeeper(key); }
public static NimbusInfo toNimbusInfo(Participant participant) { String id = participant.getId(); if (StringUtils.isBlank(id)) { throw new RuntimeException("No nimbus leader participant host found, have you started your nimbus hosts?"); } NimbusInfo nimbusInfo = NimbusInfo.parse(id); nimbusInfo.setLeader(participant.isLeader()); return nimbusInfo; }
protected ILeaderElector zkLeaderElectorImpl(Map<String, Object> conf, CuratorFramework zk, BlobStore blobStore, final TopoCache tc, IStormClusterState clusterState, List<ACL> acls, StormMetricsRegistry metricsRegistry) throws UnknownHostException { List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); String leaderLockPath = "/leader-lock"; String id = NimbusInfo.fromConf(conf).toHostPortString(); AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id)); AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference = new AtomicReference<>(leaderLatchListenerImpl( new LeaderListenerCallback(conf, zk, leaderLatchAtomicReference.get(), blobStore, tc, clusterState, acls, metricsRegistry))); return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference, leaderLatchListenerAtomicReference, blobStore, tc, clusterState, acls, metricsRegistry); }
private boolean checkIfStateContainsCurrentNimbusHost(List<String> stateInfoList, NimbusInfo nimbusInfo) { boolean containsNimbusHost = false; for (String stateInfo : stateInfoList) { if (stateInfo.contains(nimbusInfo.getHost())) { containsNimbusHost = true; break; } } return containsNimbusHost; }
@Override public void setupBlob(String key, NimbusInfo nimbusInfo, Integer versionInfo) { String path = ClusterUtils.blobstorePath(key) + ClusterUtils.ZK_SEPERATOR + nimbusInfo.toHostPortString() + "-" + versionInfo; LOG.info("set-path: {}", path); stateStorage.mkdirs(ClusterUtils.blobstorePath(key), defaultAcls); stateStorage.delete_node_blobstore(ClusterUtils.blobstorePath(key), nimbusInfo.toHostPortString()); stateStorage.set_ephemeral_node(path, null, defaultAcls); }
public static Set<NimbusInfo> getNimbodesWithLatestSequenceNumberOfBlob(CuratorFramework zkClient, String key) throws Exception { List<String> stateInfoList; try { stateInfoList = zkClient.getChildren().forPath("/blobstore/" + key); } catch (KeeperException.NoNodeException e) { // there's a race condition with a delete: blobstore // this should be thrown to the caller to indicate that the key is invalid now throw new WrappedKeyNotFoundException(key); } Set<NimbusInfo> nimbusInfoSet = new HashSet<NimbusInfo>(); int latestSeqNumber = getLatestSequenceNumber(stateInfoList); LOG.debug("getNimbodesWithLatestSequenceNumberOfBlob stateInfo {} version {}", stateInfoList, latestSeqNumber); // Get the nimbodes with the latest version for (String state : stateInfoList) { BlobKeySequenceInfo sequenceInfo = normalizeNimbusHostPortSequenceNumberInfo(state); if (latestSeqNumber == Integer.parseInt(sequenceInfo.getSequenceNumber())) { nimbusInfoSet.add(NimbusInfo.parse(sequenceInfo.getNimbusHostPort())); } } LOG.debug("nimbusInfoList {}", nimbusInfoSet); return nimbusInfoSet; }
if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && !nimbusInfo.isLeader()) { if (sequenceNumbers.last() < currentSeqNumber) { return currentSeqNumber; if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && nimbusInfo.isLeader()) { incrementMaxSequenceNumber(zkClient, currentSeqNumber); return currentSeqNumber + 1;
protected ILeaderElector zkLeaderElectorImpl(Map conf, CuratorFramework zk, BlobStore blobStore) throws UnknownHostException { List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); String leaderLockPath = "/leader-lock"; String id = NimbusInfo.fromConf(conf).toHostPortString(); AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id)); AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference = new AtomicReference<>(leaderLatchListenerImpl(conf, zk, blobStore, leaderLatchAtomicReference.get())); return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference, leaderLatchListenerAtomicReference, blobStore); }
public static NimbusInfo toNimbusInfo(Participant participant) { String id = participant.getId(); if (StringUtils.isBlank(id)) { throw new RuntimeException("No nimbus leader participant host found, have you started your nimbus hosts?"); } NimbusInfo nimbusInfo = NimbusInfo.parse(id); nimbusInfo.setLeader(participant.isLeader()); return nimbusInfo; }
@Override public void setupBlobstore(String key, NimbusInfo nimbusInfo, Integer versionInfo) { String path = ClusterUtils.blobstorePath(key) + ClusterUtils.ZK_SEPERATOR + nimbusInfo.toHostPortString() + "-" + versionInfo; LOG.info("set-path: {}", path); stateStorage.mkdirs(ClusterUtils.blobstorePath(key), acls); stateStorage.delete_node_blobstore(ClusterUtils.blobstorePath(key), nimbusInfo.toHostPortString()); stateStorage.set_ephemeral_node(path, null, acls); }
public static Set<NimbusInfo> getNimbodesWithLatestSequenceNumberOfBlob(CuratorFramework zkClient, String key) throws Exception { List<String> stateInfoList; try { stateInfoList = zkClient.getChildren().forPath("/blobstore/" + key); } catch (KeeperException.NoNodeException e) { // there's a race condition with a delete: blobstore // this should be thrown to the caller to indicate that the key is invalid now throw new KeyNotFoundException(key); } Set<NimbusInfo> nimbusInfoSet = new HashSet<NimbusInfo>(); int latestSeqNumber = getLatestSequenceNumber(stateInfoList); LOG.debug("getNimbodesWithLatestSequenceNumberOfBlob stateInfo {} version {}", stateInfoList, latestSeqNumber); // Get the nimbodes with the latest version for(String state : stateInfoList) { BlobKeySequenceInfo sequenceInfo = normalizeNimbusHostPortSequenceNumberInfo(state); if (latestSeqNumber == Integer.parseInt(sequenceInfo.getSequenceNumber())) { nimbusInfoSet.add(NimbusInfo.parse(sequenceInfo.getNimbusHostPort())); } } LOG.debug("nimbusInfoList {}", nimbusInfoSet); return nimbusInfoSet; }
if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && !nimbusInfo.isLeader()) { if (sequenceNumbers.last() < currentSeqNumber) { return currentSeqNumber; if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && nimbusInfo.isLeader()) { incrementMaxSequenceNumber(zkClient, currentSeqNumber); return currentSeqNumber + 1;
@Override public NimbusSummary getLeader() throws AuthorizationException, TException { getLeaderCalls.mark(); checkAuthorization(null, null, "getClusterInfo"); List<NimbusSummary> nimbuses = stormClusterState.nimbuses(); NimbusInfo leader = leaderElector.getLeader(); for (NimbusSummary nimbusSummary : nimbuses) { if (leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port()) { nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs())); nimbusSummary.set_isLeader(true); return nimbusSummary; } } return null; }
/** * If nimbusDetails are null, the method returns without any Zookeeper calls. */ @Test public void testUpdateKeyForBlobStore_nullNimbusInfo() { BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, null); zkClientBuilder.verifyExists(false); zkClientBuilder.verifyGetChildren(false); verify(nimbusDetails, never()).getHost(); verify(conf, never()).get(anyString()); }
@Override public void run(String[] args, Map<String, Object> conf, String command) throws Exception { for (String arg: args) { System.out.println(arg + ":"); StormTopology topo; File f = new File(arg); if (f.exists()) { topo = Utils.deserialize(FileUtils.readFileToByteArray(f), StormTopology.class); } else { //assume it is a topology id final String key = ConfigUtils.masterStormCodeKey(arg); try (BlobStore store = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf), null)) { topo = Utils.deserialize(store.readBlob(key, Nimbus.NIMBUS_SUBJECT), StormTopology.class); } } System.out.println(prettyPrint(topo)); } }
public static NimbusInfo parse(String nimbusInfo) { Matcher m = HOST_PORT_PATTERN.matcher(nimbusInfo); if (!m.matches()) { throw new RuntimeException("nimbusInfo should have format of host:port, invalid string " + nimbusInfo); } return new NimbusInfo(m.group(1), Integer.valueOf(m.group(2)), false); }