/** * Extracts a unique ResourceID from the Mesos task. * * @param taskId the Mesos TaskID * @return The ResourceID for the container */ static ResourceID extractResourceID(Protos.TaskID taskId) { return new ResourceID(taskId.getValue()); }
@Override public Collection<ResourceProfile> startNewWorker(ResourceProfile resourceProfile) { Preconditions.checkArgument( ResourceProfile.UNKNOWN.equals(resourceProfile), "The YarnResourceManager does not support custom ResourceProfiles yet. It assumes that all containers have the same resources."); requestYarnContainer(); return slotsPerWorker; }
public static void main(String[] args) { try { final ParameterTool parameterTool = ParameterTool.fromArgs(args); Configuration cfg = parameterTool.getConfiguration(); TaskManagerRunner.runTaskManager(cfg, ResourceID.generate()); } catch (Throwable t) { LOG.error("Failed to start TaskManager process", t); System.exit(1); } } }
new SlotRequest(new JobID(), new AllocationID(), resourceProfile1, taskHost)); return null; }); final ResourceID taskManagerResourceId = new ResourceID(testingContainer.getId().toString()); final SlotReport slotReport = new SlotReport( new SlotStatus( new SlotID(taskManagerResourceId, 1), new ResourceProfile(10, 1, 1, 1, 0, Collections.emptyMap())));
CompletableFuture<?> registerSlotRequestFuture = resourceManager.runInMainThread(() -> { rmServices.slotManager.registerSlotRequest( new SlotRequest(new JobID(), new AllocationID(), resourceProfile1, taskHost)); return null; });
.put(ENV_FLINK_CONTAINER_ID, resourceID.getResourceIdString()); nodeManagerClient.startContainer(container, taskManagerLaunchContext);
assertEquals(resourceDirPath + "/" + Utils.KEYTAB_FILE_NAME, configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB)); assertEquals("testuser1@domain", configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL)); assertEquals("test_container_00001", tmRunner.getResourceId().toString());
/** * Extracts a unique ResourceID from the Yarn Container. * @param container The Yarn container * @return The ResourceID for the container */ static ResourceID extractResourceID(Container container) { return new ResourceID(container.getId().toString()); }
/** * Create mock RM dependencies. */ Context() throws Exception { rpcService = new TestingRpcService(); rmServices = new MockResourceManagerRuntimeServices(); // resource manager rmResourceID = ResourceID.generate(); resourceManager = new TestingYarnResourceManager( rpcService, RM_ADDRESS, rmResourceID, flinkConfig, env, rmServices.highAvailabilityServices, rmServices.heartbeatServices, rmServices.slotManager, rmServices.metricRegistry, rmServices.jobLeaderIdService, new ClusterInformation("localhost", 1234), testingFatalErrorHandler, null, mockResourceManagerClient, mockNMClient, mockJMMetricGroup); }
taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0,
public YarnWorkerNode(Container container) { Preconditions.checkNotNull(container); this.resourceID = new ResourceID(container.getId().toString()); this.container = container; }
public SerializingLongReceiver createReceiver() throws Exception { TaskManagerLocation senderLocation = new TaskManagerLocation( ResourceID.generate(), LOCAL_ADDRESS, senderEnv.getConnectionManager().getDataPort()); InputGate receiverGate = createInputGate( jobId, dataSetID, executionAttemptID, senderLocation, receiverEnv, channels); SerializingLongReceiver receiver = new SerializingLongReceiver(receiverGate, channels * partitionIds.length); receiver.start(); return receiver; }
final AllocationID allocationID = new AllocationID(); final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L);
private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) { final List<Container> containersFromPreviousAttempts = new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse); log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts); for (final Container container : containersFromPreviousAttempts) { workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container)); } }
@Override public void onContainersCompleted(final List<ContainerStatus> statuses) { runAsync(() -> { log.debug("YARN ResourceManager reported the following containers completed: {}.", statuses); for (final ContainerStatus containerStatus : statuses) { final ResourceID resourceId = new ResourceID(containerStatus.getContainerId().toString()); final YarnWorkerNode yarnWorkerNode = workerNodeMap.remove(resourceId); if (yarnWorkerNode != null) { // Container completed unexpectedly ~> start a new one requestYarnContainerIfRequired(); } // Eagerly close the connection with task manager. closeTaskManagerConnection(resourceId, new Exception(containerStatus.getDiagnostics())); } } ); }
TaskManagerRunner.runTaskManager(configuration, new ResourceID(containerId)); return null; });
final ResourceID resourceId = new ResourceID(containerID); LOG.info("ResourceID assigned for this container: {}", resourceId);
final ResourceID id = new ResourceID(status.getContainerId().toString());
final ResourceID resourceId = new ResourceID(containerID); LOG.info("ResourceID assigned for this container: {}", resourceId);
final ResourceID resourceId = new ResourceID(containerIdStr);