@SuppressWarnings("unchecked") @VisibleForTesting void processFinishedContainer(ContainerStatus container) { LOG.info("Received completed container " + container.getContainerId()); TaskAttemptId attemptID = assignedRequests.get(container.getContainerId()); if (attemptID == null) { LOG.error("Container complete event for unknown container " + container.getContainerId()); } else { pendingRelease.remove(container.getContainerId()); assignedRequests.remove(attemptID); // Send the diagnostics String diagnostic = StringInterner.weakIntern(container.getDiagnostics()); eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnostic)); // send the container completed event to Task attempt eventHandler.handle(createContainerFinishedEvent(container, attemptID)); preemptionPolicy.handleCompletedContainer(attemptID); } }
@SuppressWarnings("unchecked") private void containerAssigned(Container allocated, ContainerRequest assigned) { // Update resource requests decContainerReq(assigned); // send the container-assigned event to task attempt eventHandler.handle(new TaskAttemptContainerAssignedEvent( assigned.attemptID, allocated, applicationACLs)); assignedRequests.add(allocated, assigned.attemptID); if (LOG.isDebugEnabled()) { LOG.debug("Assigned container (" + allocated + ") " + " to task " + assigned.attemptID + " on node " + allocated.getNodeId().toString()); } }
private void preemptReducer(int hangingMapRequests) { clearAllPendingReduceRequests(); // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(getResourceLimit(), maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); }
private void preemptReducer(int hangingMapRequests) { clearAllPendingReduceRequests(); // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(getResourceLimit(), maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); }
private void preemptReducer(int hangingMapRequests) { clearAllPendingReduceRequests(); // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(getResourceLimit(), maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); }
@SuppressWarnings("unchecked") @VisibleForTesting void processFinishedContainer(ContainerStatus container) { LOG.info("Received completed container " + container.getContainerId()); TaskAttemptId attemptID = assignedRequests.get(container.getContainerId()); if (attemptID == null) { LOG.error("Container complete event for unknown container " + container.getContainerId()); } else { pendingRelease.remove(container.getContainerId()); assignedRequests.remove(attemptID); // Send the diagnostics String diagnostic = StringInterner.weakIntern(container.getDiagnostics()); eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnostic)); // send the container completed event to Task attempt eventHandler.handle(createContainerFinishedEvent(container, attemptID)); } }
@SuppressWarnings("unchecked") private void containerAssigned(Container allocated, ContainerRequest assigned) { // Update resource requests decContainerReq(assigned); // send the container-assigned event to task attempt eventHandler.handle(new TaskAttemptContainerAssignedEvent( assigned.attemptID, allocated, applicationACLs)); assignedRequests.add(allocated, assigned.attemptID); if (LOG.isDebugEnabled()) { LOG.debug("Assigned container (" + allocated + ") " + " to task " + assigned.attemptID + " on node " + allocated.getNodeId().toString()); } }
protected AssignedRequests createAssignedRequests() { return new AssignedRequests(); }
protected AssignedRequests createAssignedRequests() { return new AssignedRequests(); }
@SuppressWarnings("unchecked") private void containerAssigned(Container allocated, ContainerRequest assigned) { // Update resource requests decContainerReq(assigned); // send the container-assigned event to task attempt eventHandler.handle(new TaskAttemptContainerAssignedEvent( assigned.attemptID, allocated, applicationACLs)); assignedRequests.add(allocated, assigned.attemptID); if (LOG.isDebugEnabled()) { LOG.info("Assigned container (" + allocated + ") " + " to task " + assigned.attemptID + " on node " + allocated.getNodeId().toString()); } }
@Override public TaskAttemptId getTaskAttempt(ContainerId container) { return reqs.get(container); }
@Override protected AssignedRequests createAssignedRequests() { AssignedRequests assignedReqs = mock(AssignedRequests.class); TaskAttemptId taskAttempt = mock(TaskAttemptId.class); when(assignedReqs.get(any(ContainerId.class))).thenReturn(taskAttempt); return assignedReqs; } }