taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId); final TaskInfo.Builder bldr = TaskInfo.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.toString()));
public void addMatchedTask(SingularityMesosTaskHolder taskHolder) { LOG.trace("Accepting task {} for offers {}", taskHolder.getTask().getTaskId(), offers.stream().map(Offer::getId).collect(Collectors.toList())); acceptedTasks.add(taskHolder); // subtract task resources from offer subtractResources(taskHolder.getMesosTask().getResourcesList()); // subtract executor resources from offer, if any are defined if (taskHolder.getMesosTask().hasExecutor() && taskHolder.getMesosTask().getExecutor().getResourcesCount() > 0) { subtractResources(taskHolder.getMesosTask().getExecutor().getResourcesList()); } }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId); final TaskInfo.Builder bldr = TaskInfo.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.toString()));
final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(MesosUtils.getPortRangeResource(31010, 31011)), taskRequest, taskResources, executorResources); assertEquals(Type.MESOS, task.getMesosTask().getContainer().getType()); final Image image = task.getMesosTask().getContainer().getMesos().getImage(); assertEquals(Protos.Image.Type.DOCKER, image.getType()); assertEquals("test:image", image.getDocker().getName()); final Volume volume = task.getMesosTask().getContainer().getVolumesList().get(0); assertEquals("/testing", volume.getContainerPath()); assertEquals("/host", volume.getHostPath()); assertEquals("iops", dockerVolume.getDriverOptions().getParameterList().get(0).getKey()); final NetworkInfo networkInfo = task.getMesosTask().getContainer().getNetworkInfosList().get(0); assertEquals("network-name", networkInfo.getName()); assertEquals(Arrays.asList("blue", "purple"), networkInfo.getGroupsList());
org.apache.mesos.v1.Protos.TaskInfo.Builder subBuilder = null; if (((bitField0_ & 0x00000001) == 0x00000001)) { subBuilder = task_.toBuilder();
return sink( sleep(frameworkId, ids, tasks), () -> tasks.forEach(task -> state.put(task.getTaskId(), TaskState.TASK_STAGING)), (e) -> LOGGER.warn("", e) );
assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); assertEquals("wat", task.getMesosTask().getCommand().getArguments(0)); assertFalse(task.getMesosTask().getCommand().getShell()); assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType()); assertEquals("docker-image", task.getMesosTask().getContainer().getDocker().getImage()); assertTrue(task.getMesosTask().getContainer().getDocker().getPrivileged()); assertEquals("/container", task.getMesosTask().getContainer().getVolumes(0).getContainerPath()); assertEquals("/host", task.getMesosTask().getContainer().getVolumes(0).getHostPath()); assertEquals(Mode.RW, task.getMesosTask().getContainer().getVolumes(0).getMode()); assertTrue(task.getMesosTask().getContainer().getDocker().getParametersList().contains(envParameter)); assertEquals(String.format("/container/%s/%s", task.getTask().getTaskRequest().getDeploy().getRequestId(), task.getTask().getTaskRequest().getDeploy().getId()), task.getMesosTask().getContainer().getVolumes(1).getContainerPath()); assertEquals(String.format("/host/%s", task.getMesosTask().getTaskId().getValue()), task.getMesosTask().getContainer().getVolumes(1).getHostPath()); assertEquals(Mode.RO, task.getMesosTask().getContainer().getVolumes(1).getMode()); assertEquals(80, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getContainerPort()); assertEquals(8080, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getHostPort()); assertEquals("tcp", task.getMesosTask().getContainer().getDocker().getPortMappings(0).getProtocol()); assertTrue(MesosUtils.getAllPorts(task.getMesosTask().getResourcesList()).contains(8080L)); assertEquals(81, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getContainerPort()); assertEquals(31000, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getHostPort()); assertEquals("udp", task.getMesosTask().getContainer().getDocker().getPortMappings(1).getProtocol()); assertEquals(Protos.ContainerInfo.DockerInfo.Network.BRIDGE, task.getMesosTask().getContainer().getDocker().getNetwork());
) { final String sleepSeconds = Optional.ofNullable(System.getenv("SLEEP_SECONDS")).orElse("15"); return TaskInfo.newBuilder() .setName(taskId) .setTaskId(
@Test public void testEnvironmentVariableOverrides() { Map<String, String> overrideVariables = new HashMap<>(); overrideVariables.put("MY_NEW_ENV_VAR", "test"); overrideVariables.put("STARTED_BY_USER", "notTestUser"); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId("test", "1", 0, 1, PendingType.IMMEDIATE, 0)) .setUser(user) .setEnvOverrides(overrideVariables) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final TaskInfo task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources).getMesosTask(); Map<String, String> environmentVariables = task .getCommand() .getEnvironment() .getVariablesList() .stream() .collect(Collectors.toMap(Variable::getName, Variable::getValue)); for (String key : overrideVariables.keySet()) { assertEquals( "Environment variable " + key + " not overridden.", environmentVariables.get(key), overrideVariables.get(key)); } }
@Test public void testAutomaticPortMapping() { NetworkConfiguration netConf = new NetworkConfiguration(); netConf.setDefaultPortMapping(true); configuration.setNetworkConfiguration(netConf); taskResources = new Resources(1, 1, 2); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityContainerInfo containerInfo = new SingularityContainerInfo( SingularityContainerType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerInfo("docker-image", false, SingularityDockerNetworkType.BRIDGE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()))); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setContainerInfo(Optional.of(containerInfo)) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(MesosUtils.getPortRangeResource(31010, 31011)), taskRequest, taskResources, executorResources); assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType()); assertEquals(Protos.ContainerInfo.DockerInfo.Network.BRIDGE, task.getMesosTask().getContainer().getDocker().getNetwork()); List<PortMapping> portMappings = task.getMesosTask().getContainer().getDocker().getPortMappingsList(); assertEquals(2, portMappings.size()); assertEquals(31010, portMappings.get(0).getHostPort()); assertEquals(31010, portMappings.get(0).getContainerPort()); assertEquals(31011, portMappings.get(1).getHostPort()); assertEquals(31011, portMappings.get(1).getContainerPort()); }
/** * <code>required .mesos.v1.TaskInfo task = 1;</code> */ public Builder mergeTask(org.apache.mesos.v1.Protos.TaskInfo value) { if (taskBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && task_ != null && task_ != org.apache.mesos.v1.Protos.TaskInfo.getDefaultInstance()) { task_ = org.apache.mesos.v1.Protos.TaskInfo.newBuilder(task_).mergeFrom(value).buildPartial(); } else { task_ = value; } onChanged(); } else { taskBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
protected SingularityTask prepTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, boolean separateHosts, Optional<String> runId) { SingularityPendingTask pendingTask = buildPendingTask(request, deploy, launchTime, instanceNo, runId); SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); Offer offer; if (separateHosts) { offer = createOffer(125, 1024, 2048, String.format("slave%s", instanceNo), String.format("host%s", instanceNo)); } else { offer = createOffer(125, 1024, 2048); } SingularityTaskId taskId = new SingularityTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, offer.getHostname(), "rack1"); TaskID taskIdProto = TaskID.newBuilder().setValue(taskId.toString()).build(); TaskInfo taskInfo = TaskInfo.newBuilder() .setAgentId(offer.getAgentId()) .setExecutor(ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("executorID"))) .setTaskId(taskIdProto) .setName("name") .build(); SingularityTask task = new SingularityTask(taskRequest, taskId, Collections.singletonList(mesosProtosUtils.offerFromProtos(offer)), mesosProtosUtils.taskFromProtos(taskInfo), Optional.of("rack1")); taskManager.savePendingTask(pendingTask); return task; }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
@Test public void testJobUserPassedAsEnvironmentVariable() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); List<Variable> environmentVariables = task.getMesosTask() .getCommand() .getEnvironment() .getVariablesList(); boolean success = false; for (Variable environmentVariable : environmentVariables) { success = success || (environmentVariable.getName().equals("STARTED_BY_USER") && environmentVariable.getValue().equals(user)); } assertTrue("Expected env variable STARTED_BY_USER to be set to " + user, success); }
@Test public void testDockerMinimalNetworking() { taskResources = new Resources(1, 1, 0, 0); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityContainerInfo containerInfo = new SingularityContainerInfo( SingularityContainerType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.NONE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()))); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setContainerInfo(Optional.of(containerInfo)) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.emptyList(), taskRequest, taskResources, executorResources); assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType()); assertEquals(Protos.ContainerInfo.DockerInfo.Network.NONE, task.getMesosTask().getContainer().getDocker().getNetwork()); }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.v1.executor.Protos.Event.Launch)) { return super.equals(obj); } org.apache.mesos.v1.executor.Protos.Event.Launch other = (org.apache.mesos.v1.executor.Protos.Event.Launch) obj; boolean result = true; result = result && (hasTask() == other.hasTask()); if (hasTask()) { result = result && getTask() .equals(other.getTask()); } result = result && unknownFields.equals(other.unknownFields); return result; }
@Test public void testArgumentCommand() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo")) .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); assertEquals("wat", task.getMesosTask().getCommand().getArguments(0)); assertFalse(task.getMesosTask().getCommand().getShell()); }
@Test public void testShellCommand() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo hi", task.getMesosTask().getCommand().getValue()); assertEquals(0, task.getMesosTask().getCommand().getArgumentsCount()); assertTrue(task.getMesosTask().getCommand().getShell()); }
public void addMatchedTask(SingularityMesosTaskHolder taskHolder) { LOG.trace("Accepting task {} for offers {}", taskHolder.getTask().getTaskId(), offers.stream().map(Offer::getId).collect(Collectors.toList())); acceptedTasks.add(taskHolder); // subtract task resources from offer subtractResources(taskHolder.getMesosTask().getResourcesList()); // subtract executor resources from offer, if any are defined if (taskHolder.getMesosTask().hasExecutor() && taskHolder.getMesosTask().getExecutor().getResourcesCount() > 0) { subtractResources(taskHolder.getMesosTask().getExecutor().getResourcesList()); } }