public static void main(String[] args) throws Exception { if(args.length != 1) { System.out.println("Need 1 arguments, but have " + Integer.toString(args.length)); System.out.println("listLocalFs <local_blobstore_dir>"); System.out.println("Migrates blobs from LocalFsBlobStore to HdfsBlobStore"); System.out.println("Example: listLocalFs '/srv/storm'"); System.exit(1); } Map<String, Object> lfsConf = Utils.readStormConfig(); lfsConf.put(Config.BLOBSTORE_DIR, args[0]); lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); /* CREATE THE BLOBSTORE */ LocalFsBlobStore lfsBlobStore = new LocalFsBlobStore(); lfsBlobStore.prepare(lfsConf, null, NimbusInfo.fromConf(lfsConf), null); /* LOOK AT HDFS BLOBSTORE */ System.out.println("Listing Local blobstore keys."); MigratorMain.listBlobStoreKeys(lfsBlobStore, null); System.out.println("Done Listing Local blobstore keys."); lfsBlobStore.shutdown(); } }
@Override public void run(String[] args, Map<String, Object> conf, String command) throws Exception { for (String arg: args) { System.out.println(arg + ":"); StormTopology topo; File f = new File(arg); if (f.exists()) { topo = Utils.deserialize(FileUtils.readFileToByteArray(f), StormTopology.class); } else { //assume it is a topology id final String key = ConfigUtils.masterStormCodeKey(arg); try (BlobStore store = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf), null)) { topo = Utils.deserialize(store.readBlob(key, Nimbus.NIMBUS_SUBJECT), StormTopology.class); } } System.out.println(prettyPrint(topo)); } }
protected ILeaderElector zkLeaderElectorImpl(Map<String, Object> conf, CuratorFramework zk, BlobStore blobStore, final TopoCache tc, IStormClusterState clusterState, List<ACL> acls, StormMetricsRegistry metricsRegistry) throws UnknownHostException { List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); String leaderLockPath = "/leader-lock"; String id = NimbusInfo.fromConf(conf).toHostPortString(); AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id)); AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference = new AtomicReference<>(leaderLatchListenerImpl( new LeaderListenerCallback(conf, zk, leaderLatchAtomicReference.get(), blobStore, tc, clusterState, acls, metricsRegistry))); return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference, leaderLatchListenerAtomicReference, blobStore, tc, clusterState, acls, metricsRegistry); }
lfsBlobStore.prepare(lfsConf, null, NimbusInfo.fromConf(lfsConf), null);
@Override public void run(String[] args, Map<String, Object> conf, String command) throws Exception { try (BlobStore nimbusBlobStore = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf), null)) { IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); Set<String> blobStoreTopologyIds = nimbusBlobStore.filterAndListKeys(key -> ConfigUtils.getIdFromBlobKey(key)); Set<String> activeTopologyIds = new HashSet<>(stormClusterState.activeStorms()); Sets.SetView<String> diffTopology = Sets.difference(activeTopologyIds, blobStoreTopologyIds); LOG.info("active-topology-ids [{}] blob-topology-ids [{}] diff-topology [{}]", activeTopologyIds, blobStoreTopologyIds, diffTopology); for (String corruptId : diffTopology) { stormClusterState.removeStorm(corruptId); } } }
private void setUpNimbusInfo(List<ACL> acls) { String leaderInfoPath = conf.get(Config.STORM_ZOOKEEPER_ROOT) + ClusterUtils.LEADERINFO_SUBTREE; NimbusInfo nimbusInfo = NimbusInfo.fromConf(conf); if (ClientZookeeper.existsNode(zk, leaderInfoPath, false)) { ClientZookeeper.setData(zk, leaderInfoPath, Utils.javaSerialize(nimbusInfo)); } else { ClientZookeeper.createNode(zk, leaderInfoPath, Utils.javaSerialize(nimbusInfo), CreateMode.PERSISTENT, acls); } }
BlobStore bs = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf), null); try { Subject nimbusSubject = new Subject();
hostPortInfo = NimbusInfo.fromConf(conf);
protected ILeaderElector zkLeaderElectorImpl(Map conf, CuratorFramework zk, BlobStore blobStore) throws UnknownHostException { List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); String leaderLockPath = "/leader-lock"; String id = NimbusInfo.fromConf(conf).toHostPortString(); AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id)); AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference = new AtomicReference<>(leaderLatchListenerImpl(conf, zk, blobStore, leaderLatchAtomicReference.get())); return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference, leaderLatchListenerAtomicReference, blobStore); }
BlobStore bs = Utils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf)); try { Subject nimbusSubject = new Subject();