public SingularityDeployBuilder toBuilder() { return new SingularityDeployBuilder(requestId, id) .setCommand(command) .setArguments(copyOfList(arguments)) .setResources(resources) .setContainerInfo(containerInfo) .setCustomExecutorCmd(customExecutorCmd) .setCustomExecutorId(customExecutorId) .setCustomExecutorSource(customExecutorSource) .setCustomExecutorResources(customExecutorResources) .setHealthcheckUri(healthcheckUri) .setHealthcheckIntervalSeconds(healthcheckIntervalSeconds) .setHealthcheckTimeoutSeconds(healthcheckTimeoutSeconds) .setHealthcheckPortIndex(healthcheckPortIndex) .setSkipHealthchecksOnDeploy(skipHealthchecksOnDeploy) .setHealthcheckProtocol(healthcheckProtocol) .setHealthcheckMaxRetries(healthcheckMaxRetries) .setHealthcheckMaxTotalTimeoutSeconds(healthcheckMaxTotalTimeoutSeconds) .setHealthcheck(healthcheck) .setConsiderHealthyAfterRunningForSeconds(considerHealthyAfterRunningForSeconds) .setDeployHealthTimeoutSeconds(deployHealthTimeoutSeconds) .setServiceBasePath(serviceBasePath) .setLoadBalancerGroups(copyOfSet(loadBalancerGroups)) .setLoadBalancerPortIndex(loadBalancerPortIndex) .setLoadBalancerOptions(copyOfMap(loadBalancerOptions)) .setLoadBalancerDomains(copyOfSet(loadBalancerDomains)) .setLoadBalancerAdditionalRoutes(copyOfList(loadBalancerAdditionalRoutes)) .setLoadBalancerTemplate(loadBalancerTemplate) .setLoadBalancerUpstreamGroup(loadBalancerUpstreamGroup) .setLoadBalancerServiceIdOverride(loadBalancerServiceIdOverride)
protected SingularityDeployMarker initSecondDeploy() { secondDeployMarker = new SingularityDeployMarker(requestId, secondDeployId, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); secondDeploy = new SingularityDeployBuilder(requestId, secondDeployId).setCommand(Optional.of("sleep 100")).build(); deployManager.saveDeploy(request, secondDeployMarker, secondDeploy); startDeploy(secondDeployMarker, System.currentTimeMillis()); return secondDeployMarker; }
protected void initLoadBalancedDeploy() { SingularityDeployBuilder builder = new SingularityDeployBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .setServiceBasePath(Optional.of("/basepath")) .setLoadBalancerGroups(Optional.of(Collections.singleton("test"))); firstDeploy = initAndFinishDeploy(request, builder, Optional.absent()); }
private SingularityDeploy checkImmediateRunDeploy(SingularityRequest request, SingularityDeploy deploy, SingularityRunNowRequest runNowRequest, List<SingularityTaskId> activeTasks, List<SingularityPendingTaskId> pendingTasks) { if (!request.isScheduled() && !request.isOneOff()) { throw badRequest("Can not request an immediate run of a non-scheduled / always running request (%s)", request); } return deploy.toBuilder() .setRunImmediately(Optional.of(fillRunNowRequest(Optional.of(runNowRequest)))) .build(); }
@Test public void testArgumentCommand() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo")) .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); assertEquals("wat", task.getMesosTask().getCommand().getArguments(0)); assertFalse(task.getMesosTask().getCommand().getShell()); }
protected void deploy(String deployId, Optional<Boolean> unpauseOnDeploy, Optional<Integer> deployRate, Optional<Boolean> autoAdvance, boolean loadBalanced) { SingularityDeployBuilder builder = new SingularityDeployBuilder(requestId, deployId); builder .setCommand(Optional.of("sleep 1")) .setDeployInstanceCountPerStep(deployRate) .setAutoAdvanceDeploySteps(autoAdvance) .setDeployStepWaitTimeMs(Optional.of(0)); if (loadBalanced) { Set<String> groups = new HashSet<>(Arrays.asList("group")); builder .setServiceBasePath(Optional.of("/basepath")) .setLoadBalancerGroups(Optional.of(groups)); } deployResource.deploy(new SingularityDeployRequest(builder.build(), unpauseOnDeploy, Optional.absent()), singularityUser); }
@Test public void testDockerMinimalNetworking() { taskResources = new Resources(1, 1, 0, 0); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityContainerInfo containerInfo = new SingularityContainerInfo( SingularityContainerType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.NONE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()))); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setContainerInfo(Optional.of(containerInfo)) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.emptyList(), taskRequest, taskResources, executorResources); assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType()); assertEquals(Protos.ContainerInfo.DockerInfo.Network.NONE, task.getMesosTask().getContainer().getDocker().getNetwork()); }
@Test(expected = WebApplicationException.class) public void whenDeployNotOneOffOrScheduledItForbidsRunImmediately() { String requestId = "request"; String deployID = "deploy"; SingularityRequest request = new SingularityRequestBuilder(requestId, RequestType.WORKER) .build(); Optional<SingularityRunNowRequest> runNowRequest = Optional.of(runNowRequest()); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, deployID) .setCommand(Optional.of("printenv")) .setRunImmediately(runNowRequest) .build(); validator.checkDeploy(request, deploy, Collections.emptyList(), Collections.emptyList()); }
protected SingularityDeploy startFirstDeploy() { firstDeploy = initDeploy(new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")), System.currentTimeMillis()); return firstDeploy; }
protected SingularityDeploy deployRequest(SingularityRequest request, double cpus, double memoryMb) { Resources r = new Resources(cpus, memoryMb, 0); SingularityDeploy deploy = new SingularityDeployBuilder(request.getId(), "deploy1") .setCommand(Optional.of("sleep 1")) .setResources(Optional.of(r)) .build(); deployResource.deploy(new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent()), singularityUser); return deploy; }
@Test public void testDeployWithImmediateRunIsLaunchedImmediately() { initRequestWithType(RequestType.SCHEDULED, false); String deployId = "d1"; SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder().setMessage("Message").build(); SingularityDeploy deploy = new SingularityDeployBuilder(requestId, deployId) .setRunImmediately(Optional.of(runNowRequest)) .setCommand(Optional.of("printenv > tmp.txt")) .build(); SingularityDeployRequest deployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent()); deployResource.deploy(deployRequest, singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); Assert.assertEquals(0, taskManager.getNumScheduledTasks()); SingularityTaskId taskId = taskManager.getActiveTaskIdsForDeploy(requestId, deployId).get(0); SingularityTask task = taskManager.getTask(taskId).get(); Map<String, Object> command = (Map<String, Object>) task.getMesosTask().getAllOtherFields().get("command"); Assert.assertEquals("printenv > tmp.txt", (String) command.get("value")); }
@Test public void testStartupTimeout() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "startup_timeout_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).setStartupTimeoutSeconds(Optional.of((int) TimeUnit.MINUTES.toSeconds(30))).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.DAYS.toMillis(1))); SingularityDeploy deploy = initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, hourAgo, hourAgo + 1, 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.<Integer>absent(), Optional.of(1000L), hourAgo + 1, Optional.<String> absent(), Optional.of("ERROR"), task.getTaskId(), Optional.of(true))); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
protected void initHCDeploy() { firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")).setHealthcheck(Optional.of(new HealthcheckOptionsBuilder("http://uri").build())), Optional.absent()); }
@Test public void testNewTaskCheckerRespectsDeployHealthcheckRetries() { initRequest(); final String deployId = "new_task_healthcheck"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(1)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); SingularityDeploy deploy = initAndFinishDeploy(request, db, Optional.absent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); Assert.assertEquals(CheckTaskState.CHECK_IF_HEALTHCHECK_OVERDUE, newTaskChecker.getTaskState(task, requestManager.getRequest(requestId), healthchecker)); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); Assert.assertEquals(CheckTaskState.CHECK_IF_HEALTHCHECK_OVERDUE, newTaskChecker.getTaskState(task, requestManager.getRequest(requestId), healthchecker)); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); Assert.assertEquals(CheckTaskState.UNHEALTHY_KILL_TASK, newTaskChecker.getTaskState(task, requestManager.getRequest(requestId), healthchecker)); }
@Test public void testPortNumber() { try { setConfigurationForNoDelay(); initRequest(); HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortNumber(Optional.of(81L)).setStartupDelaySeconds(Optional.of(0)).build(); firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId) .setCommand(Optional.of("sleep 100")).setResources(Optional.of(new Resources(1, 64, 3, 0))) .setHealthcheck(Optional.of(options)), Optional.absent()); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); scheduler.drainPendingQueue(); String[] portRange = {"80:82"}; sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1", Optional.<String> absent(), Collections.<String, String> emptyMap(), portRange))); SingularityTaskId firstTaskId = taskManager.getActiveTaskIdsForRequest(requestId).get(0); SingularityTask firstTask = taskManager.getTask(firstTaskId).get(); statusUpdate(firstTask, TaskState.TASK_RUNNING); newTaskChecker.enqueueNewTaskCheck(firstTask, requestManager.getRequest(requestId), healthchecker); Awaitility.await("healthcheck present").atMost(5, TimeUnit.SECONDS).until(() -> taskManager.getLastHealthcheck(firstTask.getTaskId()).isPresent()); Assert.assertTrue(taskManager.getLastHealthcheck(firstTask.getTaskId()).get().toString().contains("host1:81")); } finally { unsetConfigurationForNoDelay(); } }
protected SingularityDeploy initDeploy(SingularityDeployBuilder builder, long timestamp) { SingularityDeployMarker marker = new SingularityDeployMarker(requestId, builder.getId(), timestamp, Optional.<String> absent(), Optional.<String> absent()); builder.setCommand(Optional.of("sleep 100")); SingularityDeploy deploy = builder.build(); deployManager.saveDeploy(request, marker, deploy); startDeploy(marker, timestamp); return deploy; }
@Test public void itForbidsHealthCheckGreaterThanMaxTotalHealthCheck() { singularityConfiguration.setHealthcheckMaxTotalTimeoutSeconds(Optional.of(100)); createValidator(); // Total wait time on this request is (startup time) + ((interval) + (http timeout)) * (1 + retries) // = 50 + (5 + 5) * (9 + 1) // = 150 HealthcheckOptions healthCheck = new HealthcheckOptionsBuilder("/") .setPortNumber(Optional.of(8080L)) .setStartupTimeoutSeconds(Optional.of(50)) .setIntervalSeconds(Optional.of(5)) .setResponseTimeoutSeconds(Optional.of(5)) .setMaxRetries(Optional.of(9)) .build(); SingularityDeploy deploy = SingularityDeploy .newBuilder("1234567", "1234567") .setHealthcheck(Optional.of(healthCheck)) .setCommand(Optional.of("sleep 100;")) .build(); SingularityRequest request = new SingularityRequestBuilder("1234567", RequestType.SERVICE).build(); WebApplicationException exn = (WebApplicationException) catchThrowable(() -> validator.checkDeploy(request, deploy, Collections.emptyList(), Collections.emptyList())); assertThat((String) exn.getResponse().getEntity()) .contains("Max healthcheck time"); }
@Test public void testRemovedRequestData() { long now = System.currentTimeMillis(); initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent()); requestManager.markDeleted(request, now, Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); }
Optional.<String>absent())); firstDeploy = initDeploy(new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")).setHealthcheckUri(Optional.of("http://uri")), System.currentTimeMillis());
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }