protected List<String> getShardUrls() throws IOException { try { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); List<String> baseUrls = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); baseUrls.add(url); } return baseUrls; } catch (Exception e) { throw new IOException(e); } }
private List<String> getShardUrls() throws IOException { try { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); List<String> baseUrls = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); baseUrls.add(url); } return baseUrls; } catch (Exception e) { throw new IOException(e); } }
private static void logCollectionStateSummary(String collectionName, ZkStateReader zkStateReader) { Pattern portPattern = Pattern.compile(".*:([0-9]*).*"); DocCollection docCollection = zkStateReader.getClusterState().getCollection(collectionName); if (docCollection == null) { monkeyLog("Could not find collection {}", collectionName); } StringBuilder builder = new StringBuilder(); builder.append("Collection status: {"); for (Slice slice:docCollection.getSlices()) { builder.append(slice.getName() + ": {"); for (Replica replica:slice.getReplicas()) { log.info(replica.toString()); java.util.regex.Matcher m = portPattern.matcher(replica.getBaseUrl()); m.find(); String jettyPort = m.group(1); builder.append(String.format(Locale.ROOT, "%s(%s): {state: %s, type: %s, leader: %s, Live: %s}, ", replica.getName(), jettyPort, replica.getState(), replica.getType(), (replica.get("leader")!= null), zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()))); } if (slice.getReplicas().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}, "); } if (docCollection.getSlices().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}"); monkeyLog(builder.toString()); }
final boolean isActive = replica.getState() == Replica.State.ACTIVE; if (!live || !isActive) {
protected static void assertAllActive(String collection, ZkStateReader zkStateReader) throws KeeperException, InterruptedException { zkStateReader.forceUpdateCollection(collection); ClusterState clusterState = zkStateReader.getClusterState(); final DocCollection docCollection = clusterState.getCollectionOrNull(collection); if (docCollection == null || docCollection.getSlices() == null) { throw new IllegalArgumentException("Cannot find collection:" + collection); } Map<String,Slice> slices = docCollection.getSlicesMap(); for (Map.Entry<String,Slice> entry : slices.entrySet()) { Slice slice = entry.getValue(); if (slice.getState() != Slice.State.ACTIVE) { fail("Not all shards are ACTIVE - found a shard " + slice.getName() + " that is: " + slice.getState()); } Map<String,Replica> shards = slice.getReplicasMap(); for (Map.Entry<String,Replica> shard : shards.entrySet()) { Replica replica = shard.getValue(); if (replica.getState() != Replica.State.ACTIVE) { fail("Not all replicas are ACTIVE - found a replica " + replica.getName() + " that is: " + replica.getState()); } } } }
List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) shuffler.add(replica);
+ " live:" + clusterState.liveNodesContain(shard.getValue().getNodeName())); final Replica.State state = shard.getValue().getState(); if ((state == Replica.State.RECOVERING || state == Replica.State.DOWN || state == Replica.State.RECOVERY_FAILED) && clusterState.liveNodesContain(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) {
public static void verifyReplicaStatus(ZkStateReader reader, String collection, String shard, String coreNodeName, Replica.State expectedState) throws InterruptedException { int maxIterations = 100; Replica.State coreState = null; while(maxIterations-->0) { final DocCollection docCollection = reader.getClusterState().getCollectionOrNull(collection); if(docCollection != null && docCollection.getSlice(shard)!=null) { Slice slice = docCollection.getSlice(shard); Replica replica = slice.getReplicasMap().get(coreNodeName); if (replica != null) { coreState = replica.getState(); if(coreState == expectedState) { return; } } } Thread.sleep(50); } fail("Illegal state, was: " + coreState + " expected:" + expectedState + " clusterState:" + reader.getClusterState()); }
+ " live:" + clusterState.liveNodesContain(shard.getValue().getNodeName())); final Replica.State state = shard.getValue().getState(); if ((state == Replica.State.RECOVERING || state == Replica.State.DOWN || state == Replica.State.RECOVERY_FAILED) && clusterState.liveNodesContain(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) {
final boolean isActive = replica.getState() == Replica.State.ACTIVE; if (!live || !isActive) {
private void getPersistedCheckpoints() throws IOException { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(checkpointCollection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); OUTER: for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())){ HttpSolrClient httpClient = streamContext.getSolrClientCache().getHttpSolrClient(replica.getCoreUrl()); try { SolrDocument doc = httpClient.getById(id); if(doc != null) { List<String> checkpoints = (List<String>)doc.getFieldValue("checkpoint_ss"); for (String checkpoint : checkpoints) { String[] pair = checkpoint.split("~"); this.checkpoints.put(pair[0], Long.parseLong(pair[1])); } } } catch (Exception e) { throw new IOException(e); } break OUTER; } } } }
List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) shuffler.add(replica);
public static int getLiveAndActiveReplicaCount(ZkStateReader zkStateReader, String collection) { Collection<Slice> slices; slices = zkStateReader.getClusterState().getActiveSlices(collection); int liveAndActive = 0; for (Slice slice : slices) { for (Replica replica : slice.getReplicas()) { boolean live = zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()); boolean active = replica.getState() == Replica.State.ACTIVE; if (live && active) { liveAndActive++; } } } return liveAndActive; }
private static void addMissingReplicas(SolrCloudManager cloudManager, Suggestion.Ctx ctx) throws IOException { cloudManager.getClusterStateProvider().getClusterState().forEachCollection(coll -> coll.forEach(slice -> { ReplicaCount replicaCount = new ReplicaCount(); slice.forEach(replica -> { if (replica.getState() == Replica.State.ACTIVE || replica.getState() == Replica.State.RECOVERING) { replicaCount.increment(replica.getType()); } }); addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.NRT, ctx); addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.PULL, ctx); addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.TLOG, ctx); } )); }
public static int getLiveAndActiveReplicaCount(ZkStateReader zkStateReader, String collection) { Slice[] slices; slices = zkStateReader.getClusterState().getCollection(collection).getActiveSlicesArr(); int liveAndActive = 0; for (Slice slice : slices) { for (Replica replica : slice.getReplicas()) { boolean live = zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()); boolean active = replica.getState() == Replica.State.ACTIVE; if (live && active) { liveAndActive++; } } } return liveAndActive; }
static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader, TimeOut timeOut) throws Exception { log.info("Will wait for a node to become leader for {} secs", timeOut.timeLeft(SECONDS)); ZkStateReader zkStateReader = cloudClient.getZkStateReader(); zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION); for (; ; ) { ClusterState clusterState = zkStateReader.getClusterState(); DocCollection coll = clusterState.getCollection("collection1"); Slice slice = coll.getSlice(shardName); if (slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getLeader().getState() == Replica.State.ACTIVE) { log.info("Old leader {}, new leader {}. New leader got elected in {} ms", oldLeader, slice.getLeader(),timeOut.timeElapsed(MILLISECONDS) ); break; } if (timeOut.hasTimedOut()) { Diagnostics.logThreadDumps("Could not find new leader in specified timeout"); zkStateReader.getZkClient().printLayoutToStdOut(); fail("Could not find new leader even after waiting for " + timeOut.timeElapsed(MILLISECONDS) + "ms"); } Thread.sleep(100); } }
if (replica.getState() == Replica.State.ACTIVE && clusterState.getLiveNodes().contains(replica.getNodeName())) { randomActiveReplica = replica;
params.set("rows", 1); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { String coreUrl = replica.getCoreUrl(); SolrStream solrStream = new SolrStream(coreUrl, params);