@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { ElasticsearchTableLayoutHandle layoutHandle = (ElasticsearchTableLayoutHandle) layout; ElasticsearchTableHandle tableHandle = layoutHandle.getTable(); ElasticsearchTableDescription table = client.getTable(tableHandle.getSchemaName(), tableHandle.getTableName()); verify(table != null, "Table no longer exists: %s", tableHandle.toString()); List<String> indices = client.getIndices(table); ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (String index : indices) { ClusterSearchShardsResponse response = client.getSearchShards(index, table); DiscoveryNode[] nodes = response.getNodes(); for (ClusterSearchShardsGroup group : response.getGroups()) { int nodeIndex = group.getShardId().getId() % nodes.length; ElasticsearchSplit split = new ElasticsearchSplit( index, table.getType(), group.getShardId().getId(), nodes[nodeIndex].getHostName(), nodes[nodeIndex].getAddress().getPort(), layoutHandle.getTupleDomain()); splits.add(split); } } return new FixedSplitSource(splits.build()); } }
public List<TransportAddress> transportAddresses() { List<TransportAddress> lstBuilder = new ArrayList<>(); for (DiscoveryNode listedNode : listedNodes) { lstBuilder.add(listedNode.getAddress()); } return Collections.unmodifiableList(lstBuilder); }
public SendRequestTransportException(DiscoveryNode node, String action, Throwable cause) { super(node == null ? null : node.getName(), node == null ? null : node.getAddress(), action, cause); }
public ConnectTransportException(DiscoveryNode node, String msg, String action, Throwable cause) { super(node == null ? null : node.getName(), node == null ? null : node.getAddress(), action, msg, cause); this.node = node; }
public ReceiveTimeoutTransportException(DiscoveryNode node, String action, String msg) { super(node.getName(), node.getAddress(), action, msg, null); }
protected RemoteTransportException wrapInRemote(Exception e) { if (e instanceof RemoteTransportException) { return (RemoteTransportException) e; } return new RemoteTransportException(localNode.getName(), localNode.getAddress(), action, e); }
public TransportClientNodesService removeTransportAddress(TransportAddress transportAddress) { synchronized (mutex) { if (closed) { throw new IllegalStateException("transport client is closed, can't remove an address"); } List<DiscoveryNode> listNodesBuilder = new ArrayList<>(); for (DiscoveryNode otherNode : listedNodes) { if (!otherNode.getAddress().equals(transportAddress)) { listNodesBuilder.add(otherNode); } else { logger.debug("removing address [{}] from listed nodes", otherNode); } } listedNodes = Collections.unmodifiableList(listNodesBuilder); List<DiscoveryNode> nodesBuilder = new ArrayList<>(); for (DiscoveryNode otherNode : nodes) { if (!otherNode.getAddress().equals(transportAddress)) { nodesBuilder.add(otherNode); } else { logger.debug("disconnecting from node with address [{}]", otherNode); transportService.disconnectFromNode(otherNode); } } nodes = Collections.unmodifiableList(nodesBuilder); nodesSampler.sample(); } return this; }
boolean found = false; for (DiscoveryNode otherNode : listedNodes) { if (otherNode.getAddress().equals(transportAddress)) { found = true; logger.debug("address [{}] already exists with [{}], ignoring...", transportAddress, otherNode);
/** * Get a node by its address * * @param address {@link TransportAddress} of the wanted node * @return node identified by the given address or <code>null</code> if no such node exists */ public DiscoveryNode findByAddress(TransportAddress address) { for (ObjectCursor<DiscoveryNode> cursor : nodes.values()) { DiscoveryNode node = cursor.value; if (node.getAddress().equals(address)) { return node; } } return null; }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(getId()); builder.field("name", getName()); builder.field("ephemeral_id", getEphemeralId()); builder.field("transport_address", getAddress().toString()); builder.startObject("attributes"); for (Map.Entry<String, String> entry : attributes.entrySet()) { builder.field(entry.getKey(), entry.getValue()); } builder.endObject(); builder.endObject(); return builder; }
/** * Generates X-Content for a {@link DiscoveryNode} that leaves off some of the non-critical fields. */ public static XContentBuilder discoveryNodeToXContent(DiscoveryNode node, boolean outerObjectWritten, XContentBuilder builder) throws IOException { builder.field(outerObjectWritten ? "id" : "node_id", node.getId()); builder.field(outerObjectWritten ? "name" : "node_name", node.getName()); builder.field("transport_address", node.getAddress().toString()); if (node.getAttributes().isEmpty() == false) { builder.startObject(outerObjectWritten ? "attributes" : "node_attributes"); for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) { builder.field(entry.getKey(), entry.getValue()); } builder.endObject(); } return builder; }
public Connection getOrConnect(DiscoveryNode node) throws IOException { Connection result; try (Releasable ignore = connectionLock.acquire(node.getAddress())) { result = tempConnections.get(node.getAddress()); if (result == null) { ensureOpen(); boolean success = false; logger.trace("[{}] opening connection to [{}]", id(), node); result = transportService.openConnection(node, connectionProfile); try { transportService.handshake(result, connectionProfile.getHandshakeTimeout().millis()); synchronized (this) { // acquire lock and check if closed, to prevent leaving an open connection after closing ensureOpen(); Connection existing = tempConnections.put(node.getAddress(), result); assert existing == null; success = true; } } finally { if (success == false) { logger.trace("[{}] closing connection to [{}] due to failure", id(), node); IOUtils.closeWhileHandlingException(result); } } } } return result; }
/** * Checks that a node can be safely added to this node collection. * * @return null if all is OK or an error message explaining why a node can not be added. * * Note: if this method returns a non-null value, calling {@link #add(DiscoveryNode)} will fail with an * exception */ private String validateAdd(DiscoveryNode node) { for (ObjectCursor<DiscoveryNode> cursor : nodes.values()) { final DiscoveryNode existingNode = cursor.value; if (node.getAddress().equals(existingNode.getAddress()) && node.getId().equals(existingNode.getId()) == false) { return "can't add node " + node + ", found existing node " + existingNode + " with same address"; } if (node.getId().equals(existingNode.getId()) && node.equals(existingNode) == false) { return "can't add node " + node + ", found existing node " + existingNode + " with the same id but is a different node instance"; } } return null; }
builder.field("transport_address", node.getAddress().toString()); builder.field("host", node.getHostName()); builder.field("ip", node.getAddress());
protected void sendPings(final TimeValue timeout, final PingingRound pingingRound) { final ClusterState lastState = contextProvider.clusterState(); final UnicastPingRequest pingRequest = new UnicastPingRequest(pingingRound.id(), timeout, createPingResponse(lastState)); List<TransportAddress> temporalAddresses = temporalResponses.stream().map(pingResponse -> { assert clusterName.equals(pingResponse.clusterName()) : "got a ping request from a different cluster. expected " + clusterName + " got " + pingResponse.clusterName(); return pingResponse.node().getAddress(); }).collect(Collectors.toList()); final Stream<TransportAddress> uniqueAddresses = Stream.concat(pingingRound.getSeedAddresses().stream(), temporalAddresses.stream()).distinct(); // resolve what we can via the latest cluster state final Set<DiscoveryNode> nodesToPing = uniqueAddresses .map(address -> { DiscoveryNode foundNode = lastState.nodes().findByAddress(address); if (foundNode != null && transportService.nodeConnected(foundNode)) { return foundNode; } else { return new DiscoveryNode( address.toString(), address, emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()); } }).collect(Collectors.toSet()); nodesToPing.forEach(node -> sendPingRequestToNode(node, timeout, pingingRound, pingRequest)); }
seedAddresses.add(masterNode.value.getAddress());
nodeWithInfo.getHostName(), nodeWithInfo.getHostAddress(), listedNode.getAddress(), nodeWithInfo.getAttributes(), nodeWithInfo.getRoles(), nodeWithInfo.getVersion()));
private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoResponse nodesInfo) { boolean fullId = req.paramAsBoolean("full_id", false); DiscoveryNodes nodes = state.getState().nodes(); Table table = getTableWithHeader(req); for (DiscoveryNode node : nodes) { NodeInfo info = nodesInfo.getNodesMap().get(node.getId()); for (Map.Entry<String, String> attrEntry : node.getAttributes().entrySet()) { table.startRow(); table.addCell(node.getName()); table.addCell(fullId ? node.getId() : Strings.substring(node.getId(), 0, 4)); table.addCell(info == null ? null : info.getProcess().getId()); table.addCell(node.getHostName()); table.addCell(node.getHostAddress()); table.addCell(node.getAddress().address().getPort()); table.addCell(attrEntry.getKey()); table.addCell(attrEntry.getValue()); table.endRow(); } } return table; } }
private ClusterState.Builder becomeMasterAndTrimConflictingNodes(ClusterState currentState, List<DiscoveryNode> joiningNodes) { assert currentState.nodes().getMasterNodeId() == null : currentState; DiscoveryNodes currentNodes = currentState.nodes(); DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes); nodesBuilder.masterNodeId(currentState.nodes().getLocalNodeId()); for (final DiscoveryNode joiningNode : joiningNodes) { final DiscoveryNode nodeWithSameId = nodesBuilder.get(joiningNode.getId()); if (nodeWithSameId != null && nodeWithSameId.equals(joiningNode) == false) { logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameId, joiningNode); nodesBuilder.remove(nodeWithSameId.getId()); } final DiscoveryNode nodeWithSameAddress = currentNodes.findByAddress(joiningNode.getAddress()); if (nodeWithSameAddress != null && nodeWithSameAddress.equals(joiningNode) == false) { logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameAddress, joiningNode); nodesBuilder.remove(nodeWithSameAddress.getId()); } } // now trim any left over dead nodes - either left there when the previous master stepped down // or removed by us above ClusterState tmpState = ClusterState.builder(currentState).nodes(nodesBuilder).blocks(ClusterBlocks.builder() .blocks(currentState.blocks()) .removeGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)).build(); return ClusterState.builder(allocationService.deassociateDeadNodes(tmpState, false, "removed dead nodes on election")); }
private void buildRow(Table table, boolean fullId, boolean detailed, DiscoveryNodes discoveryNodes, TaskInfo taskInfo) { table.startRow(); String nodeId = taskInfo.getTaskId().getNodeId(); DiscoveryNode node = discoveryNodes.get(nodeId); table.addCell(taskInfo.getId()); table.addCell(taskInfo.getAction()); table.addCell(taskInfo.getTaskId().toString()); if (taskInfo.getParentTaskId().isSet()) { table.addCell(taskInfo.getParentTaskId().toString()); } else { table.addCell("-"); } table.addCell(taskInfo.getType()); table.addCell(taskInfo.getStartTime()); table.addCell(FORMATTER.format(Instant.ofEpochMilli(taskInfo.getStartTime()))); table.addCell(taskInfo.getRunningTimeNanos()); table.addCell(TimeValue.timeValueNanos(taskInfo.getRunningTimeNanos()).toString()); // Node information. Note that the node may be null because it has left the cluster between when we got this response and now. table.addCell(fullId ? nodeId : Strings.substring(nodeId, 0, 4)); table.addCell(node == null ? "-" : node.getHostAddress()); table.addCell(node.getAddress().address().getPort()); table.addCell(node == null ? "-" : node.getName()); table.addCell(node == null ? "-" : node.getVersion().toString()); if (detailed) { table.addCell(taskInfo.getDescription()); } table.endRow(); }