@Override public void nodesUnregistered(Set<CoordinationProtos.NodeEndpoint> unregisteredNodes) { synchronized(taskLeaderLock) { if (taskLeaderNode == null) { logger.warn("Receiving unregistration notice for {}, but no TaskLeader for {} was registered", Joiner.on(",") .join(unregisteredNodes.stream() .map(input -> String.format("%s:%d", input.getAddress(), input.getFabricPort())).collect(Collectors.toList())), taskName); return; } if (unregisteredNodes.contains(taskLeaderNode)) { taskLeaderNode = null; taskLeaderUp = false; taskLeaderLock.notifyAll(); } } }
@Override public Iterator<Object> getIterator(SabotContext sContext, OperatorContext context) { final CoordinationProtos.NodeEndpoint endpoint = sContext.getEndpoint(); final Iterable<TaskPool.ThreadInfo> threadInfos = sContext.getWorkStatsProvider().get().getSlicingThreads(); return (Iterator<Object>) (Object) StreamSupport.stream(threadInfos.spliterator(), false) .map((info) -> new SlicingThreadInfo( endpoint.getAddress(), endpoint.getFabricPort(), info )).iterator(); } },
@Override public void nodesRegistered(Set<CoordinationProtos.NodeEndpoint> registeredNodes) { Iterator<CoordinationProtos.NodeEndpoint> iterator = registeredNodes.iterator(); if (!iterator.hasNext()) { logger.warn("Received empty node registration"); return; } CoordinationProtos.NodeEndpoint endpoint = iterator.next(); synchronized(taskLeaderLock) { if (taskLeaderNode != null && !taskLeaderNode.equals(endpoint)) { logger.info("TaskLeader node for {} changed. Previous was {}:{}, new is {}:{}", taskName, taskLeaderNode.getAddress(), taskLeaderNode.getFabricPort(), endpoint.getAddress(), endpoint.getFabricPort()); } else { logger.info("New TaskLeader node for {} {}:{} registered itself.", taskName, endpoint.getAddress(), endpoint.getFabricPort()); } taskLeaderNode = endpoint; taskLeaderUp = true; taskLeaderLock.notifyAll(); } }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.CoordinationProtos.NodeEndpoint message) throws java.io.IOException { if(message.hasAddress()) output.writeString(1, message.getAddress(), false); if(message.hasUserPort()) output.writeInt32(2, message.getUserPort(), false); if(message.hasFabricPort()) output.writeInt32(3, message.getFabricPort(), false); if(message.hasRoles()) output.writeObject(5, message.getRoles(), com.dremio.exec.proto.SchemaCoordinationProtos.Roles.WRITE, false); if(message.hasStartTime()) output.writeInt64(101, message.getStartTime(), false); if(message.hasProvisionId()) output.writeString(102, message.getProvisionId(), false); if(message.hasMaxDirectMemory()) output.writeInt64(103, message.getMaxDirectMemory(), false); if(message.hasAvailableCores()) output.writeInt32(104, message.getAvailableCores(), false); } public boolean isInitialized(com.dremio.exec.proto.CoordinationProtos.NodeEndpoint message)
private void communicateChange(SourceConfig config, RpcType rpcType) { final Set<NodeEndpoint> endpoints = new HashSet<>(); endpoints.add(context.get().getEndpoint()); List<RpcFuture<Ack>> futures = new ArrayList<>(); SourceWrapper wrapper = SourceWrapper.newBuilder().setBytes(ByteString.copyFrom(ProtobufIOUtil.toByteArray(config, SourceConfig.getSchema(), LinkedBuffer.allocate()))).build(); for(NodeEndpoint e : Iterables.concat(this.context.get().getCoordinators(), this.context.get().getExecutors())) { if(!endpoints.add(e)) { continue; } SendSource send = new SendSource(wrapper, rpcType); tunnelFactory.getCommandRunner(e.getAddress(), e.getFabricPort()).runCommand(send);; logger.trace("Sending [{}] to {}:{}", config.getName(), e.getAddress(), e.getUserPort()); futures.add(send.getFuture()); } try { Futures.successfulAsList(futures).get(CHANGE_COMMUNICATION_WAIT, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e1) { logger.warn("Failure while communicating source change [{}].", config.getName(), e1); } }
@Override public Object next() { if (!beforeFirst) { throw new IllegalStateException(); } beforeFirst = false; final MemoryInfo memoryInfo = new MemoryInfo(); final NodeEndpoint endpoint = dbContext.getEndpoint(); memoryInfo.hostname = endpoint.getAddress(); memoryInfo.fabric_port = endpoint.getFabricPort(); final MemoryUsage heapMemoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); memoryInfo.heap_current = heapMemoryUsage.getUsed(); memoryInfo.heap_max = heapMemoryUsage.getMax(); BufferPoolMXBean directBean = getDirectBean(); memoryInfo.jvm_direct_current = directBean.getMemoryUsed(); memoryInfo.direct_current = dbContext.getAllocator().getAllocatedMemory(); memoryInfo.direct_max = VM.getMaxDirectMemory(); return memoryInfo; }
@Override public Object next() { ThreadInfo currentThread = threadInfoIterator.next(); final NodeEndpoint endpoint = dbContext.getEndpoint(); final long id = currentThread.getThreadId(); return new ThreadSummary(endpoint.getAddress(), endpoint.getFabricPort(), currentThread.getThreadName(), currentThread.getThreadId(), currentThread.isInNative(), currentThread.isSuspended(), currentThread.getThreadState().name(), stats.getCpuTrailingAverage(id, 1), stats.getUserTrailingAverage(id, 1), VM.availableProcessors(), getStackTrace(currentThread)); }
public ReflectionTunnel getTunnel(CoordinationProtos.NodeEndpoint ep){ return new ReflectionTunnel(ep, factory.getCommandRunner(ep.getAddress(), ep.getFabricPort())); }
public ExecTunnel getTunnel(NodeEndpoint endpoint) { return new ExecTunnel(factory.getCommandRunner(endpoint.getAddress(), endpoint.getFabricPort())); } }
public CoordTunnel getTunnel(NodeEndpoint ep){ return new CoordTunnel(ep, factory.getCommandRunner(ep.getAddress(), ep.getFabricPort())); } }
public CoordToExecTunnel getTunnel(NodeEndpoint ep){ return new CoordToExecTunnel(ep, factory.getCommandRunner(ep.getAddress(), ep.getFabricPort())); } }
private SendEndpoint<DLGetSourceRequest, DLGetSourceResponse> newGetSourceEndpoint() throws RpcException { final NodeEndpoint master = masterEndpoint.get(); if (master == null) { throw new RpcException("master node is down"); } // TODO(DX-10861): separate server-side and client-side code, when the ticket is resolved return getSourceEndpointCreator.getEndpoint(master.getAddress(), master.getFabricPort()); }
private SendEndpoint<DLFindRequest, DLFindResponse> newFindEndpoint() throws RpcException { final NodeEndpoint master = masterEndpoint.get(); if (master == null) { throw new RpcException("master node is down"); } // TODO(DX-10861): separate server-side and client-side code, when the ticket is resolved return findEndpointCreator.getEndpoint(master.getAddress(), master.getFabricPort()); }
private <Req extends MessageLite, Resp extends MessageLite, T extends SendEndpointCreator<Req, Resp>> SendEndpoint<Req, Resp> newEndpoint(T creator) throws RpcException { NodeEndpoint masterNode = master.get(); if (masterNode == null) { throw new RpcException("master node is down"); } return creator.getEndpoint(masterNode.getAddress(), masterNode.getFabricPort()); }
/** * Creates a {@link RemoteNodeFileSystem} instance. * * Only exposed for unit testing */ @VisibleForTesting FileSystem newRemoteFileSystem(final NodeEndpoint endpoint) throws IOException { final FabricCommandRunner runner = runnerFactory.getCommandRunner(endpoint.getAddress(), endpoint.getFabricPort()); RemoteNodeFileSystem rdfs = new RemoteNodeFileSystem(runner, allocator); rdfs.initialize(URI.create(format("sabot://%s:%d", endpoint.getAddress(), endpoint.getFabricPort())), getConf()); return rdfs; }
private RemoteNodeFileSystem newRemoteNodeFileSystem(Configuration configuration) throws IOException { RemoteNodeFileSystem fs = new RemoteNodeFileSystem(runner, allocator); fs.initialize(URI.create(String.format("sabot://%s:%d/", REMOTE_ENDPOINT.getAddress(), REMOTE_ENDPOINT.getFabricPort())), configuration); return fs; }
private SendEndpoint<DLGetSourcesRequest, DLGetSourcesResponse> newGetSourcesEndpoint() throws RpcException { final NodeEndpoint master = masterEndpoint.get(); if (master == null) { throw new RpcException("master node is down"); } // TODO(DX-10861): separate server-side and client-side code, when the ticket is resolved return getSourcesEndpointCreator.getEndpoint(master.getAddress(), master.getFabricPort()); }
private SendEndpoint<SearchRPC.SearchQueryRequest, SearchRPC.SearchQueryResponse> newFindEndpoint() throws RpcException { final NodeEndpoint master = masterEndpoint.get(); if (master == null) { throw new RpcException("master node is down"); } return findEndpointCreator.getEndpoint(master.getAddress(), master.getFabricPort()); }
public ExecToCoordTunnel getTunnel(NodeEndpoint identity){ Preconditions.checkArgument(ClusterCoordinator.Role.fromEndpointRoles(identity.getRoles()).contains(ClusterCoordinator.Role.COORDINATOR), "SabotNode %s is not a coordinator node.", identity); return new ExecToCoordTunnel(identity, factory.getCommandRunner(identity.getAddress(), identity.getFabricPort())); } }
@Override public void failed(final RpcException ex) { if (latch != null) { // this block only applies to intermediate fragments fragmentSubmitFailures.addFailure(endpoint, ex); latch.countDown(); } else { // this block only applies to leaf fragments // since this won't be waited on, we can wait to deliver this event once the AttemptManager is ready exception.addException(new RpcException(String.format("Failure sending leaf fragment to %s:%d.", endpoint.getAddress(), endpoint.getFabricPort()), ex)); } }