toRetain.add(node.getPublishAddress()); List<String> ddNodes = SettingsUtils.discoveredOrDeclaredNodes(settings); log.debug(String.format("Filtered discovered only nodes %s to data-only %s", SettingsUtils.discoveredOrDeclaredNodes(settings), ddNodes)); message += String.format("; node discovery is disabled and none of nodes specified fit the criterion %s", SettingsUtils.discoveredOrDeclaredNodes(settings)); SettingsUtils.setDiscoveredNodes(settings, ddNodes); } finally { bootstrap.close();
@Override public void setSettings(Settings settings) { emptyAsNull = settings.getReadFieldEmptyAsNull(); richDate = settings.getMappingDateRich(); arrayInclude = SettingsUtils.getFieldArrayFilterInclude(settings); arrayExclude = StringUtils.tokenize(settings.getReadFieldAsArrayExclude()); } }
public static List<NodeInfo> discoverNodesIfNeeded(Settings settings, Log log) { if (settings.getNodesDiscovery()) { RestClient bootstrap = new RestClient(settings); try { List<NodeInfo> discoveredNodes = bootstrap.getHttpNodes(false); if (log.isDebugEnabled()) { log.debug(String.format("Nodes discovery enabled - found %s", discoveredNodes)); } SettingsUtils.addDiscoveredNodes(settings, discoveredNodes); return discoveredNodes; } finally { bootstrap.close(); } } return null; }
private static List<String> qualifyNodes(String nodes, int defaultPort, boolean resolveHostNames) { List<String> list = StringUtils.tokenize(nodes); for (int i = 0; i < list.size(); i++) { String nodeIp = (resolveHostNames ? resolveHostToIpIfNecessary(list.get(i)) : list.get(i)); list.set(i, qualifyNode(nodeIp, defaultPort)); } return list; }
public static void addDiscoveredNodes(Settings settings, List<NodeInfo> discoveredNodes) { // clean-up and merge Set<String> nodes = new LinkedHashSet<String>(); nodes.addAll(declaredNodes(settings)); for (NodeInfo node : discoveredNodes) { if (node.hasHttp()) { nodes.add(node.getPublishAddress()); } } setDiscoveredNodes(settings, nodes); }
public NetworkClient(Settings settings, TransportFactory transportFactory) { this.settings = settings.copy(); this.secureSettings = new SecureSettings(settings); this.nodes = SettingsUtils.discoveredOrDeclaredNodes(settings); this.transportFactory = transportFactory; // shuffle the list of nodes so in case of failures, the fallback is spread Collections.shuffle(nodes); if (SettingsUtils.hasPinnedNode(settings)) { // move pinned node in front to be selected (only once) String pinnedNode = SettingsUtils.getPinnedNode(settings); if (log.isDebugEnabled()) { log.debug("Opening (pinned) network client to " + pinnedNode); } nodes.remove(pinnedNode); nodes.add(0, pinnedNode); } selectNextNode(); Assert.notNull(currentTransport, "no node information provided"); }
if (!SettingsUtils.hasPinnedNode(settings) && partition.getLocations().length > 0) { String pinAddress = checkLocality(partition.getLocations(), log); if (pinAddress != null) { log.debug(String.format("Partition reader instance [%s] assigned to [%s]", partition, pinAddress)); SettingsUtils.pinNode(settings, pinAddress); partition, clientNode)); SettingsUtils.pinNode(settings, clientNode); .size(settings.getScrollSize()) .limit(settings.getScrollLimit()) .fields(SettingsUtils.determineSourceFields(settings)) .filters(QueryUtils.parseFilters(settings)) .shard(Integer.toString(partition.getShardId()))
InitializationUtils.filterNonIngestNodesIfNeeded(settings, log); List<String> nodes = SettingsUtils.discoveredOrDeclaredNodes(settings); SettingsUtils.pinNode(settings, nodes.get(selectedNode));
String node = SettingsUtils.getPinnedNode(settings); if (log.isDebugEnabled()) { log.debug(String.format("Partition writer instance [%s] assigned to [%s]", currentInstance, node)); SettingsUtils.pinNode(settings, targetNode.getPublishAddress()); String node = SettingsUtils.getPinnedNode(settings); repository = new RestRepository(settings);
public static PartitionReader createReader(Settings settings, PartitionDefinition partition, Log log) { if (!SettingsUtils.hasPinnedNode(settings)) { if (log.isDebugEnabled()) { log.debug(String.format("Partition reader instance [%s] assigned to [%s]:[%s]", partition, SettingsUtils.pinNode(settings, partition.nodeIp, partition.nodePort); partition, clientNode)); SettingsUtils.pinNode(settings, clientNode); .node(partition.nodeId).restrictToNode(partition.onlyNode && (!settings.getNodesClientOnly() && !settings.getNodesWANOnly())); queryBuilder.fields(settings.getScrollFields()); queryBuilder.filter(SettingsUtils.getFilters(settings));
protected List<List<Map<String, Object>>> doGetReadTargetShards() { return client.targetShards(resources.getResourceRead().index(), SettingsUtils.getFixedRouting(settings)); }
/** * Creates a RestRepository for use with a multi-index resource pattern. The client is left pinned * to the original node that it was pinned to since the shard locations cannot be determined at all. * @param settings Job settings * @param currentInstance Partition number * @param resource Configured write resource * @param log Logger to use * @return The RestRepository to be used by the partition writer */ private static RestRepository initMultiIndices(Settings settings, long currentInstance, Resource resource, Log log) { if (log.isDebugEnabled()) { log.debug(String.format("Resource [%s] resolves as an index pattern", resource)); } // multi-index write - since we don't know before hand what index will be used, use an already selected node String node = SettingsUtils.getPinnedNode(settings); if (log.isDebugEnabled()) { log.debug(String.format("Partition writer instance [%s] assigned to [%s]", currentInstance, node)); } return new RestRepository(settings); }
@Test public void testHostWithoutAPortFallingBackToDefaultAndNoDiscoveryWithSchema() throws Exception { Properties props = new Properties(); props.setProperty("es.nodes", "http://localhost"); props.setProperty("es.port", "9800"); props.setProperty("es.nodes.discovery", "false"); PropertiesSettings settings = new PropertiesSettings(props); List<String> nodes = SettingsUtils.discoveredOrDeclaredNodes(settings); assertThat(nodes.size(), equalTo(1)); assertThat("http://127.0.0.1:9800", equalTo(nodes.get(0))); }
protected Object[] doGetReadTargetShards(boolean clientNodesOnly) { List<List<Map<String, Object>>> info = client.targetShards(resourceR.index(), SettingsUtils.getFixedRouting(settings)); Map<Shard, Node> shards = new LinkedHashMap<Shard, Node>(); List<String> nodes = SettingsUtils.discoveredOrDeclaredNodes(settings); Random rnd = new Random();
private boolean selectNextNode() { if (nextClient >= nodes.size()) { return false; } if (currentTransport != null) { stats.nodeRetries++; } closeTransport(); currentNode = nodes.get(nextClient++); SettingsUtils.pinNode(settings, currentNode); currentTransport = transportFactory.create(settings, secureSettings, currentNode); return true; }
public static List<String> discoveredOrDeclaredNodes(Settings settings) { // returned the discovered nodes or, if not defined, the set nodes String discoveredNodes = settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_DISCOVERED_NODES); return (StringUtils.hasText(discoveredNodes) ? StringUtils.tokenize(discoveredNodes) : declaredNodes(settings)); }
public static List<QueryBuilder> parseFilters(Settings settings) { String[] rawFilters = SettingsUtils.getFilters(settings); if (rawFilters == null) { return Collections.emptyList(); } List<QueryBuilder> filters = new ArrayList<QueryBuilder>(); for (String filter : rawFilters) { filter = filter.trim(); try { filters.add(SimpleQueryParser.parse(filter, false)); } catch (IOException e) { throw new IllegalArgumentException("Failed to parse filter: " + filter, e); } } return filters; }
public static void pinNode(Settings settings, String node, int port) { if (StringUtils.hasText(node) && port > 0) { settings.setProperty(InternalConfigurationOptions.INTERNAL_ES_PINNED_NODE, qualifyNode(node, port)); } }
public NetworkClient(Settings settings) { this.settings = settings.copy(); this.nodes = SettingsUtils.discoveredOrDeclaredNodes(settings); // shuffle the list of nodes so in case of failures, the fallback is spread Collections.shuffle(nodes); if (SettingsUtils.hasPinnedNode(settings)) { // move pinned node in front to be selected (only once) String pinnedNode = SettingsUtils.getPinnedNode(settings); if (log.isDebugEnabled()) { log.debug("Opening (pinned) network client to " + pinnedNode); } nodes.remove(pinnedNode); nodes.add(0, pinnedNode); } selectNextNode(); Assert.notNull(currentTransport, "no node information provided"); }
if (!SettingsUtils.hasPinnedNode(settings) && partition.getLocations().length > 0) { String pinAddress = checkLocality(partition.getLocations(), log); if (pinAddress != null) { log.debug(String.format("Partition reader instance [%s] assigned to [%s]", partition, pinAddress)); SettingsUtils.pinNode(settings, pinAddress); partition, clientNode)); SettingsUtils.pinNode(settings, clientNode); .size(settings.getScrollSize()) .limit(settings.getScrollLimit()) .fields(SettingsUtils.determineSourceFields(settings)) .filters(QueryUtils.parseFilters(settings)) .shard(Integer.toString(partition.getShardId()))