public synchronized void forceReregister(NodeControllerService ncs) throws InterruptedException { registrationCompleted = false; ncs.getExecutor().submit(() -> { try { return ncs.registerNode(this); } catch (Exception e) { LOGGER.log(Level.ERROR, "Failed registering with cc", e); throw new IllegalStateException(e); } }); while (!registrationCompleted) { wait(); } }
public synchronized void jobCompleted(JobId jobId, JobStatus status) { if (status == JobStatus.FAILURE) { failedJobsCache.put(jobId, jobId); } final List<IPartition> jobPartitions = unregisterPartitions(jobId); final List<NetworkOutputChannel> pendingRequests = removePendingRequests(jobId, status); if (!jobPartitions.isEmpty() || !pendingRequests.isEmpty()) { ncs.getExecutor().execute(() -> { jobPartitions.forEach(IDeallocatable::deallocate); pendingRequests.forEach(networkOutputChannel -> networkOutputChannel .abort(AbstractChannelWriteInterface.REMOTE_ERROR_CODE)); }); } }
public void close() { long stillAllocated = memoryAllocation.get(); if (stillAllocated > 0) { LOGGER.trace(() -> "Freeing leaked " + stillAllocated + " bytes"); serviceCtx.getMemoryManager().deallocate(stillAllocated); } nodeController.getExecutor().execute(() -> deallocatableRegistry.close()); }
@Override public void receivedMessage(IMessage message, String nodeId) throws Exception { INcAddressedMessage absMessage = (INcAddressedMessage) message; if (LOGGER.isInfoEnabled()) { LOGGER.info("Received message: " + absMessage); } ncs.getExecutor().submit(() -> { try { absMessage.handle(appContext); } catch (Exception e) { LOGGER.log(Level.WARN, "Could not process message: {}", message, e); } }); }
@Override protected void doRun() throws Exception { LOGGER.info("Aborting all tasks for controller {}", ccId); IResultPartitionManager resultPartitionManager = ncs.getResultPartitionManager(); if (resultPartitionManager == null) { LOGGER.log(Level.WARN, "ResultPartitionManager is null on " + ncs.getId()); } Deque<Task> abortedTasks = new ArrayDeque<>(); Collection<Joblet> joblets = ncs.getJobletMap().values(); // TODO(mblow): should we have one jobletmap per cc? joblets.stream().filter(joblet -> joblet.getJobId().getCcId().equals(ccId)).forEach(joblet -> { joblet.getTaskMap().values().forEach(task -> { task.abort(); abortedTasks.add(task); }); final JobId jobId = joblet.getJobId(); if (resultPartitionManager != null) { resultPartitionManager.abortReader(jobId); resultPartitionManager.sweep(jobId); } ncs.getWorkQueue().schedule(new CleanupJobletWork(ncs, jobId, JobStatus.FAILURE)); }); ncs.getExecutor().submit(new EnsureAllCcTasksCompleted(ncs, ccId, abortedTasks)); } }
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new MaterializingPipelinedPartition(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor()); } };
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new MaterializedPartitionWriter(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor()); } };
private IPartitionCollector createPartitionCollector(TaskAttemptDescriptor td, final int partition, Task task, int i, IConnectorDescriptor conn, RecordDescriptor recordDesc, IConnectorPolicy cPolicy) throws HyracksDataException { IPartitionCollector collector = conn.createPartitionCollector(task, recordDesc, partition, td.getInputPartitionCounts()[i], td.getPartitionCount()); if (cPolicy.materializeOnReceiveSide()) { return new ReceiveSideMaterializingCollector(task, ncs.getPartitionManager(), collector, task.getTaskAttemptId(), ncs.getExecutor()); } else { return collector; } }
List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(aid); task = null; task = new Task(joblet, flags, taId, han.getClass().getName(), ncs.getExecutor(), ncs, createInputChannels(td, inputs)); IOperatorNodePushable operator = han.createPushRuntime(task, rdp, partition, td.getPartitionCount());
ncs.getExecutor().submit(new ShutdownTask(sdrf.isTerminateNCService())); return; ncs.getExecutor().submit(new ThreadDumpTask(ncs, tdrf.getRequestId(), tdrf.getCcId())); return; ncs.getExecutor().submit(new PingTask(ncs, pcf.getCcId())); return; ncs.getExecutor().submit(new HeartbeatAckTask(ncs, nbaf.getCcId(), nbaf.getException())); return;
final NetworkOutputChannel networkOutputChannel = new NetworkOutputChannel(ccb, 0); final MaterializingPipelinedPartition mpp = new MaterializingPipelinedPartition(ctx, ncs.getPartitionManager(), pid, taId, ncs.getExecutor()); mpp.open();