public List<Replica> getReplicas(EnumSet<Replica.Type> s) { List<Replica> replicas = new ArrayList<>(); for (Slice slice : this) { replicas.addAll(slice.getReplicas(s)); } return replicas; }
public List<Replica> getReplicas() { List<Replica> replicas = new ArrayList<>(); for (Slice slice : this) { replicas.addAll(slice.getReplicas()); } return replicas; }
/** * Gets the list of replicas that have a type present in s */ public List<Replica> getReplicas(EnumSet<Replica.Type> s) { return this.getReplicas(r->s.contains(r.getType())); }
/** * Get a (reproducibly) random replica from a {@link Slice} */ protected static Replica getRandomReplica(Slice slice) { List<Replica> replicas = new ArrayList<>(slice.getReplicas()); if (replicas.size() == 0) fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString()); Collections.shuffle(replicas, random()); return replicas.get(0); }
/** * Get a (reproducibly) random replica from a {@link Slice} matching a predicate */ protected static Replica getRandomReplica(Slice slice, Predicate<Replica> matchPredicate) { List<Replica> replicas = new ArrayList<>(slice.getReplicas()); if (replicas.size() == 0) fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString()); Collections.shuffle(replicas, random()); for (Replica replica : replicas) { if (matchPredicate.test(replica)) return replica; } fail("Couldn't get random replica that matched conditions\n" + slice.toString()); return null; // just to keep the compiler happy - fail will always throw an Exception }
public List<List<String>> extractShardUrls(String zkHost, String collection) { DocCollection docCollection = extractDocCollection(zkHost, collection); List<Slice> slices = getSortedSlices(docCollection.getSlices()); List<List<String>> solrUrls = new ArrayList<List<String>>(slices.size()); for (Slice slice : slices) { if (slice.getLeader() == null) { throw new IllegalArgumentException("Cannot find SolrCloud slice leader. " + "It looks like not all of your shards are registered in ZooKeeper yet"); } Collection<Replica> replicas = slice.getReplicas(); List<String> urls = new ArrayList<String>(replicas.size()); for (Replica replica : replicas) { ZkCoreNodeProps props = new ZkCoreNodeProps(replica); urls.add(props.getCoreUrl()); } solrUrls.add(urls); } return solrUrls; }
protected int getTotalReplicas(String collection) { ZkStateReader zkStateReader = cloudClient.getZkStateReader(); DocCollection coll = zkStateReader.getClusterState().getCollectionOrNull(collection); if (coll == null) return 0; // support for when collection hasn't been created yet int cnt = 0; for (Slice slices : coll.getSlices()) { cnt += slices.getReplicas().size(); } return cnt; }
public String getShardId(String collectionName, String nodeName, String coreName) { Collection<CollectionRef> states = collectionStates.values(); if (collectionName != null) { CollectionRef c = collectionStates.get(collectionName); if (c != null) states = Collections.singletonList( c ); } for (CollectionRef ref : states) { DocCollection coll = ref.get(); if(coll == null) continue;// this collection go tremoved in between, skip for (Slice slice : coll.getSlices()) { for (Replica replica : slice.getReplicas()) { // TODO: for really large clusters, we could 'index' on this String rnodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP); String rcore = replica.getStr(ZkStateReader.CORE_NAME_PROP); if (nodeName.equals(rnodeName) && coreName.equals(rcore)) { return slice.getName(); } } } } return null; }
public String getShardId(String collectionName, String nodeName, String coreName) { Collection<CollectionRef> states = collectionStates.values(); if (collectionName != null) { CollectionRef c = collectionStates.get(collectionName); if (c != null) states = Collections.singletonList( c ); } for (CollectionRef ref : states) { DocCollection coll = ref.get(); if(coll == null) continue;// this collection go tremoved in between, skip for (Slice slice : coll.getSlices()) { for (Replica replica : slice.getReplicas()) { // TODO: for really large clusters, we could 'index' on this String rnodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP); String rcore = replica.getStr(ZkStateReader.CORE_NAME_PROP); if (nodeName.equals(rnodeName) && coreName.equals(rcore)) { return slice.getName(); } } } } return null; }
protected List<String> getShardUrls() throws IOException { try { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); List<String> baseUrls = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); baseUrls.add(url); } return baseUrls; } catch (Exception e) { throw new IOException(e); } }
public List<List<String>> extractShardUrls(String zkHost, String collection) { DocCollection docCollection = extractDocCollection(zkHost, collection); List<Slice> slices = getSortedSlices(docCollection.getSlices()); List<List<String>> solrUrls = new ArrayList<List<String>>(slices.size()); for (Slice slice : slices) { if (slice.getLeader() == null) { throw new IllegalArgumentException("Cannot find SolrCloud slice leader. " + "It looks like not all of your shards are registered in ZooKeeper yet"); } Collection<Replica> replicas = slice.getReplicas(); List<String> urls = new ArrayList<String>(replicas.size()); for (Replica replica : replicas) { ZkCoreNodeProps props = new ZkCoreNodeProps(replica); if (replica.getStr(Slice.LEADER) == null) { urls.add(props.getCoreUrl()); // add followers at tail } else { urls.add(0, props.getCoreUrl()); // insert leader at head } } solrUrls.add(urls); } return solrUrls; }
private List<String> getShardUrls() throws IOException { try { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); List<String> baseUrls = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); baseUrls.add(url); } return baseUrls; } catch (Exception e) { throw new IOException(e); } }
private static void logCollectionStateSummary(String collectionName, ZkStateReader zkStateReader) { Pattern portPattern = Pattern.compile(".*:([0-9]*).*"); DocCollection docCollection = zkStateReader.getClusterState().getCollection(collectionName); if (docCollection == null) { monkeyLog("Could not find collection {}", collectionName); } StringBuilder builder = new StringBuilder(); builder.append("Collection status: {"); for (Slice slice:docCollection.getSlices()) { builder.append(slice.getName() + ": {"); for (Replica replica:slice.getReplicas()) { log.info(replica.toString()); java.util.regex.Matcher m = portPattern.matcher(replica.getBaseUrl()); m.find(); String jettyPort = m.group(1); builder.append(String.format(Locale.ROOT, "%s(%s): {state: %s, type: %s, leader: %s, Live: %s}, ", replica.getName(), jettyPort, replica.getState(), replica.getType(), (replica.get("leader")!= null), zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()))); } if (slice.getReplicas().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}, "); } if (docCollection.getSlices().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}"); monkeyLog(builder.toString()); }
protected void logReplicaTypesReplicationInfo(String collectionName, ZkStateReader zkStateReader) throws KeeperException, InterruptedException, IOException { log.info("## Collecting extra Replica.Type information of the cluster"); zkStateReader.updateLiveNodes(); StringBuilder builder = new StringBuilder(); zkStateReader.forceUpdateCollection(collectionName); DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName); for(Slice s:collection.getSlices()) { Replica leader = s.getLeader(); for (Replica r:s.getReplicas()) { if (!r.isActive(zkStateReader.getClusterState().getLiveNodes())) { builder.append(String.format(Locale.ROOT, "Replica %s not in liveNodes or is not active%s", r.getName(), System.lineSeparator())); continue; } if (r.equals(leader)) { builder.append(String.format(Locale.ROOT, "Replica %s is leader%s", r.getName(), System.lineSeparator())); } logReplicationDetails(r, builder); } } log.info("Summary of the cluster: " + builder.toString()); }
protected SolrStream constructStream(String sql) throws IOException { try { ZkStateReader zkStateReader = this.connection.getClient().getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.connection.getCollection(), zkStateReader, true); List<Replica> shuffler = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); for (Replica replica : replicas) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CommonParams.QT, "/sql"); params.set("stmt", sql); for(String propertyName : this.connection.getProperties().stringPropertyNames()) { params.set(propertyName, this.connection.getProperties().getProperty(propertyName)); } Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); return new SolrStream(url, params); } catch (Exception e) { throw new IOException(e); } }
Replica leader = slice.getLeader(); if (directUpdatesToLeadersOnly && leader == null) { for (Replica replica : slice.getReplicas( replica -> replica.isActive(getClusterStateProvider().getLiveNodes()) && replica.getType() == Replica.Type.NRT)) { urls.add(url); if (!directUpdatesToLeadersOnly) { for (Replica replica : slice.getReplicas()) { if (!replica.getNodeName().equals(leader.getNodeName()) && !replica.getName().equals(leader.getName())) {
private void getPersistedCheckpoints() throws IOException { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(checkpointCollection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); OUTER: for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())){ HttpSolrClient httpClient = streamContext.getSolrClientCache().getHttpSolrClient(replica.getCoreUrl()); try { SolrDocument doc = httpClient.getById(id); if(doc != null) { List<String> checkpoints = (List<String>)doc.getFieldValue("checkpoint_ss"); for (String checkpoint : checkpoints) { String[] pair = checkpoint.split("~"); this.checkpoints.put(pair[0], Long.parseLong(pair[1])); } } } catch (Exception e) { throw new IOException(e); } break OUTER; } } } }
public static int getLiveAndActiveReplicaCount(ZkStateReader zkStateReader, String collection) { Collection<Slice> slices; slices = zkStateReader.getClusterState().getActiveSlices(collection); int liveAndActive = 0; for (Slice slice : slices) { for (Replica replica : slice.getReplicas()) { boolean live = zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()); boolean active = replica.getState() == Replica.State.ACTIVE; if (live && active) { liveAndActive++; } } } return liveAndActive; }
public static int getLiveAndActiveReplicaCount(ZkStateReader zkStateReader, String collection) { Slice[] slices; slices = zkStateReader.getClusterState().getCollection(collection).getActiveSlicesArr(); int liveAndActive = 0; for (Slice slice : slices) { for (Replica replica : slice.getReplicas()) { boolean live = zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()); boolean active = replica.getState() == Replica.State.ACTIVE; if (live && active) { liveAndActive++; } } } return liveAndActive; }
private long getCheckpoint(Slice slice, Set<String> liveNodes) throws IOException { Collection<Replica> replicas = slice.getReplicas(); long checkpoint = -1; ModifiableSolrParams params = new ModifiableSolrParams();