private static int copyDataDown(SolrZkClient zkClient, String zkPath, File file) throws IOException, KeeperException, InterruptedException { byte[] data = zkClient.getData(zkPath, null, null, true); if (data != null && data.length > 1) { // There are apparently basically empty ZNodes. log.info("Writing file {}", file.toString()); Files.write(file.toPath(), data); return data.length; } return 0; }
public void updateAliases() throws KeeperException, InterruptedException { byte[] data = zkClient.getData(ALIASES, null, null, true); Aliases aliases = ClusterState.load(data); ZkStateReader.this.aliases = aliases; } public Map getClusterProps(){
@Override public void visit(String path) throws InterruptedException, KeeperException { String finalDestination = dest; if (path.equals(source) == false) finalDestination += "/" + path.substring(source.length() + 1); zkClient.makePath(finalDestination, false, true); zkClient.setData(finalDestination, zkClient.getData(path, null, null, true), true); } }
/** * Assumes data in ZooKeeper is a JSON string, deserializes it and returns as a Map * * @param zkClient the zookeeper client * @param path the path to the znode being read * @param retryOnConnLoss whether to retry the operation automatically on connection loss, see {@link org.apache.solr.common.cloud.ZkCmdExecutor#retryOperation(ZkOperation)} * @return a Map if the node exists and contains valid JSON or an empty map if znode does not exist or has a null data */ public static Map<String, Object> getJson(SolrZkClient zkClient, String path, boolean retryOnConnLoss) throws KeeperException, InterruptedException { try { byte[] bytes = zkClient.getData(path, null, null, retryOnConnLoss); if (bytes != null && bytes.length > 0) { return (Map<String, Object>) Utils.fromJSON(bytes); } } catch (KeeperException.NoNodeException e) { return Collections.emptyMap(); } return Collections.emptyMap(); }
@Override public Map<String, Object> readIndexerProperties() { Properties props = new Properties(); try { byte[] data = zkClient.getData(path, null, null, true); if (data != null) { props.load(new StringReader(new String(data, StandardCharsets.UTF_8))); } } catch (Exception e) { SolrZkClient.checkInterrupted(e); log.warn("Could not read DIH properties from " + path + " :" + e.getClass(), e); } return propertiesToMap(props); } }
@Override public VersionedData getData(String path, Watcher watcher) throws NoSuchElementException, IOException, KeeperException, InterruptedException { Stat stat = new Stat(); try { byte[] bytes = zkClient.getData(path, watcher, stat, true); return new VersionedData(stat.getVersion(), bytes, String.valueOf(stat.getEphemeralOwner())); } catch (KeeperException.NoNodeException e) { throw new NoSuchElementException(path); } catch (InterruptedException e) { throw e; } }
@Override public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws InterruptedException, IOException { Map<String, Object> map = new HashMap<>(); Stat stat = new Stat(); try { byte[] bytes = zkClient.getData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, watcher, stat, true); if (bytes != null && bytes.length > 0) { map = (Map<String, Object>) fromJSON(bytes); } } catch (KeeperException.NoNodeException e) { // ignore } catch (KeeperException e) { throw new IOException(e); } map.put(AutoScalingParams.ZK_VERSION, stat.getVersion()); return new AutoScalingConfig(map); }
/** * Return the collection properties * @throws IOException if there is an error reading properties from zookeeper */ @SuppressWarnings("unchecked") public Map<String, String> getCollectionProperties(String collection) throws IOException { try { return (Map<String, String>) Utils.fromJSON(client.getData(ZkStateReader.getCollectionPropsPath(collection), null, new Stat(), true)); } catch (KeeperException.NoNodeException e) { return Collections.emptyMap(); } catch (KeeperException | InterruptedException e) { throw new IOException("Error reading properties for collection " + collection, SolrZkClient.checkInterrupted(e)); } }
private String checkForAlias(SolrZkClient zkClient, String collection) throws KeeperException, InterruptedException { byte[] aliasData = zkClient.getData(ZkStateReader.ALIASES, null, null, true); Aliases aliases = ClusterState.load(aliasData); String alias = aliases.getCollectionAlias(collection); if (alias != null) { List<String> aliasList = StrUtils.splitSmart(alias, ",", true); if (aliasList.size() > 1) { throw new IllegalArgumentException("collection cannot be an alias that maps to multiple collections"); } collection = aliasList.get(0); } return collection; }
/** * Return the cluster properties * @throws IOException if there is an error reading properties from the cluster */ @SuppressWarnings("unchecked") public Map<String, Object> getClusterProperties() throws IOException { try { Map<String, Object> properties = (Map<String, Object>) Utils.fromJSON(client.getData(ZkStateReader.CLUSTER_PROPS, null, new Stat(), true)); return convertCollectionDefaultsToNestedFormat(properties); } catch (KeeperException.NoNodeException e) { return Collections.emptyMap(); } catch (KeeperException | InterruptedException e) { throw new IOException("Error reading cluster property", SolrZkClient.checkInterrupted(e)); } }
private String checkForAlias(SolrZkClient zkClient, String collection) throws KeeperException, InterruptedException { byte[] aliasData = zkClient.getData(ZkStateReader.ALIASES, null, null, true); Aliases aliases = ClusterState.load(aliasData); String alias = aliases.getCollectionAlias(collection); if (alias != null) { List<String> aliasList = StrUtils.splitSmart(alias, ",", true); if (aliasList.size() > 1) { throw new IllegalArgumentException("collection cannot be an alias that maps to multiple collections"); } collection = aliasList.get(0); } return collection; }
private void downloadFromZK(String zkPath, Path dir) throws IOException { try { List<String> files = zkClient.getChildren(zkPath, null, true); Files.createDirectories(dir); for (String file : files) { List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true); if (children.size() == 0) { byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true); Path filename = dir.resolve(file); logger.info("Writing file {}", filename); Files.write(filename, data); } else { downloadFromZK(zkPath + "/" + file, dir.resolve(file)); } } } catch (KeeperException | InterruptedException e) { throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(), SolrZkClient.checkInterrupted(e)); } }
/** * Get current {@link AutoScalingConfig}. * @param watcher optional {@link Watcher} to set on a znode to watch for config changes. * @return current configuration from <code>autoscaling.json</code>. NOTE: * this data is retrieved from ZK on each call. */ public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws KeeperException, InterruptedException { Stat stat = new Stat(); Map<String, Object> map = new HashMap<>(); try { byte[] bytes = zkClient.getData(SOLR_AUTOSCALING_CONF_PATH, watcher, stat, true); if (bytes != null && bytes.length > 0) { map = (Map<String, Object>) fromJSON(bytes); } } catch (KeeperException.NoNodeException e) { // ignore } map.put(AutoScalingParams.ZK_VERSION, stat.getVersion()); return new AutoScalingConfig(map); }
@SuppressWarnings("unchecked") private void loadClusterProperties() { try { while (true) { try { byte[] data = zkClient.getData(ZkStateReader.CLUSTER_PROPS, clusterPropertiesWatcher, new Stat(), true); this.clusterProperties = ClusterProperties.convertCollectionDefaultsToNestedFormat((Map<String, Object>) Utils.fromJSON(data)); log.debug("Loaded cluster properties: {}", this.clusterProperties); return; } catch (KeeperException.NoNodeException e) { this.clusterProperties = Collections.emptyMap(); log.debug("Loaded empty cluster properties"); // set an exists watch, and if the node has been created since the last call, // read the data again if (zkClient.exists(ZkStateReader.CLUSTER_PROPS, clusterPropertiesWatcher, true) == null) return; } } } catch (KeeperException | InterruptedException e) { log.error("Error reading cluster properties from zookeeper", SolrZkClient.checkInterrupted(e)); } }
/** * Ensures the internal aliases is up to date. If there is a change, return true. * * @return true if an update was performed */ public boolean update() throws KeeperException, InterruptedException { log.debug("Checking ZK for most up to date Aliases {}", ALIASES); // Call sync() first to ensure the subsequent read (getData) is up to date. zkClient.getSolrZooKeeper().sync(ALIASES, null, null); Stat stat = new Stat(); final byte[] data = zkClient.getData(ALIASES, null, stat, true); return setIfNewer(Aliases.fromJSON(data, stat.getVersion())); }
private void copyConfigDirFromZk(String fromZkPath, String toZkPath, Set<String> copiedToZkPaths) throws IOException { try { List<String> files = zkClient.getChildren(fromZkPath, null, true); for (String file : files) { List<String> children = zkClient.getChildren(fromZkPath + "/" + file, null, true); if (children.size() == 0) { final String toZkFilePath = toZkPath + "/" + file; log.info("Copying zk node {} to {}", fromZkPath + "/" + file, toZkFilePath); byte[] data = zkClient.getData(fromZkPath + "/" + file, null, null, true); zkClient.makePath(toZkFilePath, data, true); if (copiedToZkPaths != null) copiedToZkPaths.add(toZkFilePath); } else { copyConfigDirFromZk(fromZkPath + "/" + file, toZkPath + "/" + file, copiedToZkPaths); } } } catch (KeeperException | InterruptedException e) { throw new IOException("Error copying nodes from zookeeper path " + fromZkPath + " to " + toZkPath, SolrZkClient.checkInterrupted(e)); } }
public Map getClusterProps(){ Map result = null; try { if(getZkClient().exists(ZkStateReader.CLUSTER_PROPS, true)){ result = (Map) Utils.fromJSON(getZkClient().getData(ZkStateReader.CLUSTER_PROPS, null, new Stat(), true)) ; } else { result= new LinkedHashMap(); } return result; } catch (Exception e) { throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading cluster properties",e) ; } }
@SuppressWarnings("unchecked") private Map<String, String> fetchCollectionProperties(String collection, Watcher watcher) throws KeeperException, InterruptedException { final String znodePath = getCollectionPropsPath(collection); while (true) { try { byte[] data = zkClient.getData(znodePath, watcher, null, true); return (Map<String, String>) Utils.fromJSON(data); } catch (ClassCastException e) { throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to parse collection properties for collection " + collection, e); } catch (KeeperException.NoNodeException e) { if (watcher != null) { // Leave an exists watch in place in case a collectionprops.json is created later. Stat exists = zkClient.exists(znodePath, watcher, true); if (exists != null) { // Rare race condition, we tried to fetch the data and couldn't find it, then we found it exists. // Loop and try again. continue; } } return Collections.emptyMap(); } } }
private DocCollection fetchCollectionState(String coll, Watcher watcher) throws KeeperException, InterruptedException { String collectionPath = getCollectionPath(coll); while (true) { try { Stat stat = new Stat(); byte[] data = zkClient.getData(collectionPath, watcher, stat, true); ClusterState state = ClusterState.load(stat.getVersion(), data, Collections.<String>emptySet(), collectionPath); ClusterState.CollectionRef collectionRef = state.getCollectionStates().get(coll); return collectionRef == null ? null : collectionRef.get(); } catch (KeeperException.NoNodeException e) { if (watcher != null) { // Leave an exists watch in place in case a state.json is created later. Stat exists = zkClient.exists(collectionPath, watcher, true); if (exists != null) { // Rare race condition, we tried to fetch the data and couldn't find it, then we found it exists. // Loop and try again. continue; } } return null; } } }
public static void moveZnode(SolrZkClient zkClient, String src, String dst) throws SolrServerException, KeeperException, InterruptedException { String destName = normalizeDest(src, dst, true, true); // Special handling if the source has no children, i.e. copying just a single file. if (zkClient.getChildren(src, null, true).size() == 0) { zkClient.makePath(destName, false, true); zkClient.setData(destName, zkClient.getData(src, null, null, true), true); } else { traverseZkTree(zkClient, src, VISIT_ORDER.VISIT_PRE, new ZkCopier(zkClient, src, destName)); } // Insure all source znodes are present in dest before deleting the source. // throws error if not all there so the source is left intact. Throws error if source and dest don't match. checkAllZnodesThere(zkClient, src, destName); clean(zkClient, src); }