if (log.isDebugEnabled()) { log.debug("### evaluate permissions for {} on {}", user, clusterService.localNode().getName()); log.debug("action: "+action0+" ("+request.getClass().getSimpleName()+")");
private static DiscoveryNode maybeAddProxyAddress(String proxyAddress, DiscoveryNode node) { if (proxyAddress == null || proxyAddress.isEmpty()) { return node; } else { // resovle proxy address lazy here InetSocketAddress proxyInetAddress = RemoteClusterAware.parseSeedAddress(proxyAddress); return new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), node.getHostName(), node .getHostAddress(), new TransportAddress(proxyInetAddress), node.getAttributes(), node.getRoles(), node.getVersion()); } }
public TransportClientNodesService removeTransportAddress(TransportAddress transportAddress) { synchronized (mutex) { if (closed) { throw new IllegalStateException("transport client is closed, can't remove an address"); } List<DiscoveryNode> listNodesBuilder = new ArrayList<>(); for (DiscoveryNode otherNode : listedNodes) { if (!otherNode.getAddress().equals(transportAddress)) { listNodesBuilder.add(otherNode); } else { logger.debug("removing address [{}] from listed nodes", otherNode); } } listedNodes = Collections.unmodifiableList(listNodesBuilder); List<DiscoveryNode> nodesBuilder = new ArrayList<>(); for (DiscoveryNode otherNode : nodes) { if (!otherNode.getAddress().equals(transportAddress)) { nodesBuilder.add(otherNode); } else { logger.debug("disconnecting from node with address [{}]", otherNode); transportService.disconnectFromNode(otherNode); } } nodes = Collections.unmodifiableList(nodesBuilder); nodesSampler.sample(); } return this; }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { ElasticsearchTableLayoutHandle layoutHandle = (ElasticsearchTableLayoutHandle) layout; ElasticsearchTableHandle tableHandle = layoutHandle.getTable(); ElasticsearchTableDescription table = client.getTable(tableHandle.getSchemaName(), tableHandle.getTableName()); verify(table != null, "Table no longer exists: %s", tableHandle.toString()); List<String> indices = client.getIndices(table); ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (String index : indices) { ClusterSearchShardsResponse response = client.getSearchShards(index, table); DiscoveryNode[] nodes = response.getNodes(); for (ClusterSearchShardsGroup group : response.getGroups()) { int nodeIndex = group.getShardId().getId() % nodes.length; ElasticsearchSplit split = new ElasticsearchSplit( index, table.getType(), group.getShardId().getId(), nodes[nodeIndex].getHostName(), nodes[nodeIndex].getAddress().getPort(), layoutHandle.getTupleDomain()); splits.add(split); } } return new FixedSplitSource(splits.build()); } }
@Override public boolean mustAck(DiscoveryNode discoveryNode) { // repository was created on both master and data nodes return discoveryNode.isMasterNode() || discoveryNode.isDataNode(); } });
public ConnectTransportException(DiscoveryNode node, String msg, String action, Throwable cause) { super(node == null ? null : node.getName(), node == null ? null : node.getAddress(), action, msg, cause); this.node = node; }
if(actionTrace.isTraceEnabled()) { actionTrace.trace("Node "+cs.localNode().getName()+" -> "+action+" ("+count+"): userIsAdmin="+userIsAdmin+"/conRequest="+confRequest+"/internalRequest="+internalRequest +"origin="+threadContext.getTransient(ConfigConstants.SG_ORIGIN)+"/directRequest="+HeaderHelper.isDirectRequest(threadContext)+"/remoteAddress="+request.remoteAddress()); threadContext.putHeader("_sg_trace"+System.currentTimeMillis()+"#"+UUID.randomUUID().toString(), Thread.currentThread().getName()+" FILTER -> "+"Node "+cs.localNode().getName()+" -> "+action+" userIsAdmin="+userIsAdmin+"/conRequest="+confRequest+"/internalRequest="+internalRequest +"origin="+threadContext.getTransient(ConfigConstants.SG_ORIGIN)+"/directRequest="+HeaderHelper.isDirectRequest(threadContext)+"/remoteAddress="+request.remoteAddress()+" "+threadContext.getHeaders().entrySet().stream().filter(p->!p.getKey().startsWith("_sg_trace")).collect(Collectors.toMap(p -> p.getKey(), p -> p.getValue()))); log.error("No user found for "+ action+" from "+request.remoteAddress()+" "+threadContext.getTransient(ConfigConstants.SG_ORIGIN)+" via "+threadContext.getTransient(ConfigConstants.SG_CHANNEL_TYPE)+" "+threadContext.getHeaders()); listener.onFailure(new ElasticsearchSecurityException("No user found for "+action, RestStatus.INTERNAL_SERVER_ERROR)); return;
ArrayList<DiscoveryNode> newFilteredNodes = new ArrayList<>(); for (DiscoveryNode listedNode : listedNodes) { try (Transport.Connection connection = transportService.openConnection(listedNode, LISTED_NODES_PROFILE)){ final PlainTransportFuture<LivenessResponse> handler = new PlainTransportFuture<>( new FutureTransportResponseHandler<LivenessResponse>() { transportService.sendRequest(connection, TransportLivenessAction.NAME, new LivenessRequest(), TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout).build(), handler); final LivenessResponse livenessResponse = handler.txGet(); if (!ignoreClusterName && !clusterName.equals(livenessResponse.getClusterName())) { logger.warn("node {} not part of the cluster {}, ignoring...", listedNode, clusterName); newFilteredNodes.add(listedNode); } else { newNodes.add(new DiscoveryNode(nodeWithInfo.getName(), nodeWithInfo.getId(), nodeWithInfo.getEphemeralId(), nodeWithInfo.getHostName(), nodeWithInfo.getHostAddress(), listedNode.getAddress(), nodeWithInfo.getAttributes(), nodeWithInfo.getRoles(), nodeWithInfo.getVersion())); logger.debug(() -> new ParameterizedMessage("failed to connect to node [{}], ignoring...", listedNode), e); hostFailureListener.onNodeDisconnected(listedNode, e); } catch (Exception e) { logger.info(() -> new ParameterizedMessage("failed to get node info for {}, disconnecting...", listedNode), e);
if (addedNode.isDataNode()) { dataNodeAdded = true; break; if (this.isMaster && dataNodeAdded && event.state().getNodes().getDataNodes().size() > 1) { if (logger.isDebugEnabled()) { logger.debug("data node was added, retrieving new cluster info"); if (removedNode.isDataNode()) { if (logger.isTraceEnabled()) { logger.trace("Removing node from cluster info: {}", removedNode.getId()); if (leastAvailableSpaceUsages.containsKey(removedNode.getId())) { ImmutableOpenMap.Builder<String, DiskUsage> newMaxUsages = ImmutableOpenMap.builder(leastAvailableSpaceUsages); newMaxUsages.remove(removedNode.getId()); leastAvailableSpaceUsages = newMaxUsages.build(); if (mostAvailableSpaceUsages.containsKey(removedNode.getId())) { ImmutableOpenMap.Builder<String, DiskUsage> newMinUsages = ImmutableOpenMap.builder(mostAvailableSpaceUsages); newMinUsages.remove(removedNode.getId()); mostAvailableSpaceUsages = newMinUsages.build();
ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext()); DiscoveryNode masterNode = currentState.nodes().getMasterNode(); Predicate<ClusterState> changePredicate = MasterNodeChangePredicate.build(currentState); if (masterNode == null) { logger.warn("no master known for action [{}] for shard entry [{}]", actionName, request); waitForNewMasterAndRetry(actionName, observer, request, listener, changePredicate); } else { logger.debug("sending [{}] to [{}] for shard entry [{}]", actionName, masterNode.getId(), request); transportService.sendRequest(masterNode, actionName, request, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { @Override
try (Releasable ignored = connectionLock.acquire(node.getId())) { Transport.Connection connection = connectedNodes.get(node); if (connection != null) { if (logger.isDebugEnabled()) { logger.debug("connected to node [{}]", node); } finally { if (success == false) { // close the connection if there is a failure logger.trace(() -> new ParameterizedMessage("failed to connect to [{}], cleaning dangling connections", node)); IOUtils.closeWhileHandlingException(connection);
final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndexName()); logger.debug("{} deleted shard reason [{}]", shardId, reason); if (clusterState.nodes().getLocalNode().isMasterNode() == false && canDeleteIndexContents(shardId.getIndex(), indexSettings)) { if (nodeEnv.findAllShardIds(shardId.getIndex()).isEmpty()) { logger.trace("[{}] still has shard stores, leaving as is", shardId.getIndex());
private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException { final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); final IndexShard shard = indexService.getShard(request.shardId().id()); final ShardRouting routingEntry = shard.routingEntry(); if (routingEntry.primary() == false || routingEntry.active() == false) { throw new DelayRecoveryException("source shard [" + routingEntry + "] is not an active primary"); } if (request.isPrimaryRelocation() && (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) { logger.debug("delaying recovery of {} as source shard is not marked yet as relocating to {}", request.shardId(), request.targetNode()); throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]"); } RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard); logger.trace("[{}][{}] starting recovery to {}", request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode()); try { return handler.recoverToTarget(); } finally { ongoingRecoveries.remove(shard, handler); } }
/** Updates the shard snapshot status by sending a {@link UpdateIndexShardSnapshotStatusRequest} to the master node */ void sendSnapshotShardUpdate(final Snapshot snapshot, final ShardId shardId, final ShardSnapshotStatus status, final DiscoveryNode masterNode) { try { if (masterNode.getVersion().onOrAfter(Version.V_6_1_0)) { UpdateIndexShardSnapshotStatusRequest request = new UpdateIndexShardSnapshotStatusRequest(snapshot, shardId, status); transportService.sendRequest(transportService.getLocalNode(), UPDATE_SNAPSHOT_STATUS_ACTION_NAME, request, INSTANCE_SAME); } else { UpdateSnapshotStatusRequestV6 requestV6 = new UpdateSnapshotStatusRequestV6(snapshot, shardId, status); transportService.sendRequest(masterNode, UPDATE_SNAPSHOT_STATUS_ACTION_NAME_V6, requestV6, INSTANCE_SAME); } } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to update snapshot state", snapshot, status), e); } }
private ClusterState.Builder becomeMasterAndTrimConflictingNodes(ClusterState currentState, List<DiscoveryNode> joiningNodes) { assert currentState.nodes().getMasterNodeId() == null : currentState; DiscoveryNodes currentNodes = currentState.nodes(); DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes); nodesBuilder.masterNodeId(currentState.nodes().getLocalNodeId()); for (final DiscoveryNode joiningNode : joiningNodes) { final DiscoveryNode nodeWithSameId = nodesBuilder.get(joiningNode.getId()); if (nodeWithSameId != null && nodeWithSameId.equals(joiningNode) == false) { logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameId, joiningNode); nodesBuilder.remove(nodeWithSameId.getId()); } final DiscoveryNode nodeWithSameAddress = currentNodes.findByAddress(joiningNode.getAddress()); if (nodeWithSameAddress != null && nodeWithSameAddress.equals(joiningNode) == false) { logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameAddress, joiningNode); nodesBuilder.remove(nodeWithSameAddress.getId()); } } // now trim any left over dead nodes - either left there when the previous master stepped down // or removed by us above ClusterState tmpState = ClusterState.builder(currentState).nodes(nodesBuilder).blocks(ClusterBlocks.builder() .blocks(currentState.blocks()) .removeGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)).build(); return ClusterState.builder(allocationService.deassociateDeadNodes(tmpState, false, "removed dead nodes on election")); }
public void verify(String repository, boolean readOnly, String verificationToken, final ActionListener<VerifyResponse> listener) { final DiscoveryNodes discoNodes = clusterService.state().nodes(); final DiscoveryNode localNode = discoNodes.getLocalNode(); final ObjectContainer<DiscoveryNode> masterAndDataNodes = discoNodes.getMasterAndDataNodes().values(); final List<DiscoveryNode> nodes = new ArrayList<>(); for (ObjectCursor<DiscoveryNode> cursor : masterAndDataNodes) { DiscoveryNode node = cursor.value; if (readOnly && node.getVersion().before(Version.V_6_6_0)) { continue; final AtomicInteger counter = new AtomicInteger(nodes.size()); for (final DiscoveryNode node : nodes) { if (node.equals(localNode)) { try { doVerify(repository, verificationToken, localNode); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); errors.add(new VerificationFailure(node.getId(), e)); transportService.sendRequest(node, ACTION_NAME, new VerifyNodeRepositoryRequest(repository, verificationToken), new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { @Override public void handleResponse(TransportResponse.Empty response) {
void handleJoinRequest(final DiscoveryNode node, final ClusterState state, final MembershipAction.JoinCallback callback) { if (nodeJoinController == null) { throw new IllegalStateException("discovery module is not yet started"); } else { // we do this in a couple of places including the cluster update thread. This one here is really just best effort // to ensure we fail as fast as possible. onJoinValidators.stream().forEach(a -> a.accept(node, state)); if (state.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false) { MembershipAction.ensureMajorVersionBarrier(node.getVersion(), state.getNodes().getMinNodeVersion()); } // try and connect to the node, if it fails, we can raise an exception back to the client... transportService.connectToNode(node); // validate the join request, will throw a failure if it fails, which will get back to the // node calling the join request try { membership.sendValidateJoinRequestBlocking(node, state, joinTimeout); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("failed to validate incoming join request from node [{}]", node), e); callback.onFailure(new IllegalStateException("failure when sending a validation request to node", e)); return; } nodeJoinController.handleJoinRequest(node, callback); } }
private void sendFullClusterState(ClusterState clusterState, Map<Version, BytesReference> serializedStates, DiscoveryNode node, TimeValue publishTimeout, SendingController sendingController) { BytesReference bytes = serializedStates.get(node.getVersion()); if (bytes == null) { try { bytes = serializeFullClusterState(clusterState, node.getVersion()); serializedStates.put(node.getVersion(), bytes); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("failed to serialize cluster_state before publishing it to node {}", node), e); sendingController.onNodeSendFailed(node, e); return; } } sendClusterStateToNode(clusterState, bytes, node, publishTimeout, sendingController, false, serializedStates); }
@Override public void messageReceived(final BanParentTaskRequest request, final TransportChannel channel) throws Exception { if (request.ban) { logger.debug("Received ban for the parent [{}] on the node [{}], reason: [{}]", request.parentTaskId, clusterService.localNode().getId(), request.reason); taskManager.setBan(request.parentTaskId, request.reason); } else { logger.debug("Removing ban for the parent [{}] on the node [{}]", request.parentTaskId, clusterService.localNode().getId()); taskManager.removeBan(request.parentTaskId); } channel.sendResponse(TransportResponse.Empty.INSTANCE); } }
private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard, RoutingTable routingTable, ClusterState clusterState) { final ShardRouting currentRoutingEntry = shard.routingEntry(); assert currentRoutingEntry.isSameAllocation(shardRouting) : "local shard has a different allocation id but wasn't cleaning by removeShards. " + "cluster state: " + shardRouting + " local: " + currentRoutingEntry; .filter(shr -> nodes.get(shr.currentNodeId()).getVersion().before(Version.V_6_0_0_alpha1)) .map(ShardRouting::allocationId) .map(AllocationId::getId) .collect(Collectors.toSet()); shard.updateShardState(shardRouting, primaryTerm, primaryReplicaSyncer::resync, clusterState.version(), inSyncIds, indexShardRoutingTable, pre60AllocationIds); } catch (Exception e) { if (logger.isTraceEnabled()) { logger.trace("{} master marked shard as initializing, but shard has state [{}], resending shard started to {}", shardRouting.shardId(), state, nodes.getMasterNode()); if (nodes.getMasterNode() != null) { shardStateAction.shardStarted(shardRouting, "master " + nodes.getMasterNode() + " marked shard as initializing, but shard state is [" + state + "], mark shard as started",