@Before public void setUp() throws Exception { // make zookeeper interfaces curator = zk().curatorWithSuperAuth(); final ZooKeeperClientProvider zkcp = new ZooKeeperClientProvider( new DefaultZooKeeperClient(curator), ZooKeeperModelReporter.noop()); final List<EventSender> eventSenders = Collections.emptyList(); zkMasterModel = new ZooKeeperMasterModel(zkcp, getClass().getName(), eventSenders, ""); startDefaultMaster(); agent = startDefaultAgent(TEST_HOST); client = defaultClient(); awaitHostRegistered(client, TEST_HOST, LONG_WAIT_SECONDS, SECONDS); }
final String token) throws HostNotFoundException, JobNotDeployedException, TokenVerificationException { assertHostExists(client, host); final Deployment deployment = getDeployment(host, jobId); if (deployment == null) { throw new JobNotDeployedException(host, jobId); final Job job = getJob(client, jobId); verifyToken(token, job); final String configHostJobPath = Paths.configHostJob(host, jobId); nodes.add(Paths.configJobHost(jobId, host)); final List<Integer> staticPorts = staticPorts(job); for (final int port : staticPorts) { nodes.add(Paths.configHostPort(host, port));
/** * Used to update the existing deployment of a job. */ @Override public void updateDeployment(final String host, final Deployment deployment, final String token) throws HostNotFoundException, JobNotDeployedException, TokenVerificationException { log.info("updating deployment {}: {}", deployment, host); final ZooKeeperClient client = provider.get("updateDeployment"); final JobId jobId = deployment.getJobId(); final Job job = getJob(client, jobId); final Deployment existingDeployment = getDeployment(host, jobId); if (job == null) { throw new JobNotDeployedException(host, jobId); } verifyToken(token, job); assertHostExists(client, host); assertTaskExists(client, host, deployment.getJobId()); final String path = Paths.configHostJob(host, jobId); final Task task = new Task(job, deployment.getGoal(), existingDeployment.getDeployerUser(), existingDeployment.getDeployerMaster(), existingDeployment.getDeploymentGroupName()); try { client.setData(path, task.toJsonBytes()); } catch (Exception e) { throw new HeliosRuntimeException("updating deployment " + deployment + " on host " + host + " failed", e); } }
final ZooKeeperClient client = provider.get("getJobStatus"); final Job job = getJob(client, jobId); if (job == null) { return null; hosts = listJobHosts(client, jobId); } catch (JobDoesNotExistException e) { return null; final ImmutableMap.Builder<String, TaskStatus> taskStatuses = ImmutableMap.builder(); for (final String host : hosts) { final TaskStatus taskStatus = getTaskStatus(client, host, jobId); if (taskStatus != null) { taskStatuses.put(host, taskStatus); final Deployment deployment = getDeployment(host, jobId); if (deployment != null) { deployments.put(host, deployment);
/** * Returns the current status of the host named by {@code host}. */ @Override public HostStatus getHostStatus(final String host) { final ZooKeeperClient client = provider.get("getHostStatus"); if (!ZooKeeperRegistrarUtil.isHostRegistered(client, host)) { log.warn("Host {} isn't registered in ZooKeeper.", host); return null; } final boolean up = checkHostUp(client, host); final HostInfo hostInfo = getHostInfo(client, host); final AgentInfo agentInfo = getAgentInfo(client, host); final Map<JobId, Deployment> tasks = getTasks(client, host); final Map<JobId, TaskStatus> statuses = getTaskStatuses(client, host); final Map<String, String> environment = getEnvironment(client, host); final Map<String, String> labels = getLabels(client, host); return HostStatus.newBuilder() .setJobs(tasks) .setStatuses(fromNullable(statuses).or(EMPTY_STATUSES)) .setHostInfo(hostInfo) .setAgentInfo(agentInfo) .setStatus(up ? UP : DOWN) .setEnvironment(environment) .setLabels(labels) .build(); }
final Job job = getJob(id); verifyToken(token, job); final String taskCreationPath = Paths.configHostJobCreation(host, id, operationId); final List<Integer> staticPorts = staticPorts(job); final Map<String, byte[]> portNodes = Maps.newHashMap(); final byte[] idJson = id.toJsonBytes(); } catch (NoNodeException e) { assertJobExists(client, id); assertHostExists(client, host); deployJobRetry(client, host, deployment, count + 1, token); } catch (NodeExistsException e) { checkForPortConflicts(client, host, port, id);
final DeploymentGroup deploymentGroup, final String host) { final TaskStatus taskStatus = getTaskStatus(client, host, deploymentGroup.getJobId()); final JobId jobId = deploymentGroup.getJobId(); final Deployment deployment = getDeployment(host, jobId); if (deployment == null) { return opFactory.error( if (isRolloutTimedOut(client, deploymentGroup)) { return opFactory.error("timed out while retrieving job status", host, RollingUpdateError.TIMED_OUT_RETRIEVING_JOB_STATUS); if (isRolloutTimedOut(client, deploymentGroup)) { return rollingUpdateTimedoutError(opFactory, host, jobId, taskStatus); final Deployment deployment = getDeployment(host, deploymentGroup.getJobId()); if (deployment == null) { return opFactory.error(
@Test public void testMaster() throws Exception { final JobId jobId = createAndAwaitJobRunning(); // shut down the agent so it cannot remove the tombstone we make agent.stopAsync().awaitTerminated(); // make sure things look correct before assertFalse(zkMasterModel.getJobs().isEmpty()); assertEquals(START, zkMasterModel.getDeployment(TEST_HOST, jobId).getGoal()); // undeploy job client.undeploy(jobId, TEST_HOST).get(); // These used to be filtered away assertNull(zkMasterModel.getDeployment(TEST_HOST, jobId)); assertTrue(zkMasterModel.getHostStatus(TEST_HOST).getJobs().isEmpty()); }
private RollingUpdateTaskResult rollingUpdateAwaitRunning(final DeploymentGroup deploymentGroup, final String host) { final ZooKeeperClient client = provider.get("rollingUpdateAwaitRunning"); final Map<JobId, TaskStatus> taskStatuses = getTaskStatuses(client, host); final Deployment deployment = getDeployment(host, deploymentGroup.getJobId()); if (deployment == null) { return RollingUpdateTaskResult.error( if (isRolloutTimedOut(deploymentGroup, client)) { return RollingUpdateTaskResult.error("timed out while retrieving job status", host); if (isRolloutTimedOut(deploymentGroup, client)) { final Deployment deployment = getDeployment(host, deploymentGroup.getJobId()); if (deployment == null) { return RollingUpdateTaskResult.error(
.filter(host -> checkHostUp(zooKeeperClient, host)) .collect(Collectors.toList()); final List<String> upHostsToDeploy = updateHostsCopy.stream() .filter(host -> checkHostUp(zooKeeperClient, host)) .collect(Collectors.toList());
/** * Creates a config entry within the specified agent to un/deploy a job, or more generally, change * the deployment status according to the {@code Goal} value in {@link Deployment}. */ @Override public void deployJob(final String host, final Deployment deployment, final String token) throws JobDoesNotExistException, JobAlreadyDeployedException, HostNotFoundException, JobPortAllocationConflictException, TokenVerificationException { final ZooKeeperClient client = provider.get("deployJob"); deployJobRetry(client, host, deployment, 0, token); }
@Override public void deployJob(String host, Deployment job) throws HostNotFoundException, JobAlreadyDeployedException, JobDoesNotExistException, JobPortAllocationConflictException { try { deployJob(host, job, Job.EMPTY_TOKEN); } catch (TokenVerificationException e) { throw new RuntimeException(e); } }
private RollingUpdateOp rollingUpdateDeploy(final ZooKeeperClient client, final RollingUpdateOpFactory opFactory, final DeploymentGroup deploymentGroup, final String host) { final Deployment deployment = Deployment.of(deploymentGroup.getJobId(), Goal.START, Deployment.EMTPY_DEPLOYER_USER, this.name, deploymentGroup.getName()); try { final String token = MoreObjects.firstNonNull( deploymentGroup.getRolloutOptions().getToken(), Job.EMPTY_TOKEN); return opFactory.nextTask(getDeployOperations(client, host, deployment, token)); } catch (JobDoesNotExistException e) { return opFactory.error(e, host, RollingUpdateError.JOB_NOT_FOUND); } catch (TokenVerificationException e) { return opFactory.error(e, host, RollingUpdateError.TOKEN_VERIFICATION_ERROR); } catch (HostNotFoundException e) { return opFactory.error(e, host, RollingUpdateError.HOST_NOT_FOUND); } catch (JobPortAllocationConflictException e) { return opFactory.error(e, host, RollingUpdateError.PORT_CONFLICT); } catch (JobAlreadyDeployedException e) { // Nothing to do return opFactory.nextTask(); } }
@Override public AgentInfo getAgentInfo(final String host) { return getAgentInfo(provider.get("getAgentInfo"), host); }
@Override public boolean isHostUp(final String host) { final ZooKeeperClient client = provider.get("isHostUp"); return ZooKeeperRegistrarUtil.isHostRegistered(client, host) && checkHostUp(client, host); }
final boolean up = checkHostUp(client, host); final HostInfo hostInfo = getHostInfo(client, host); final AgentInfo agentInfo = getAgentInfo(client, host); final Map<JobId, Deployment> tasks = getTasks(client, host); final Map<JobId, TaskStatus> statuses = getTaskStatuses(client, host); final Map<String, String> environment = getEnvironment(client, host); final Map<String, String> labels = getLabels(client, host);
final Job job = getJob(id); verifyToken(token, job); final String taskCreationPath = Paths.configHostJobCreation(host, id, operationId); final List<Integer> staticPorts = staticPorts(job); final Map<String, byte[]> portNodes = Maps.newHashMap(); final byte[] idJson = id.toJsonBytes(); } catch (NoNodeException e) { assertJobExists(client, id); assertHostExists(client, host); deployJobRetry(client, host, deployment, count + 1, token); } catch (NodeExistsException e) {
final ZooKeeperClient client = provider.get("getJobStatus"); final Job job = getJob(client, jobId); if (job == null) { return null; hosts = listJobHosts(client, jobId); } catch (JobDoesNotExistException e) { return null; final ImmutableMap.Builder<String, TaskStatus> taskStatuses = ImmutableMap.builder(); for (final String host : hosts) { final TaskStatus taskStatus = getTaskStatus(client, host, jobId); if (taskStatus != null) { taskStatuses.put(host, taskStatus); final Deployment deployment = getDeployment(host, jobId); if (deployment != null) { deployments.put(host, deployment);
/** * Creates a config entry within the specified agent to un/deploy a job, or more generally, change * the deployment status according to the {@code Goal} value in {@link Deployment}. */ @Override public void deployJob(final String host, final Deployment deployment, final String token) throws JobDoesNotExistException, JobAlreadyDeployedException, HostNotFoundException, JobPortAllocationConflictException, TokenVerificationException { final ZooKeeperClient client = provider.get("deployJob"); deployJobRetry(client, host, deployment, 0, token); }
@Override public void deployJob(String host, Deployment job) throws HostNotFoundException, JobAlreadyDeployedException, JobDoesNotExistException, JobPortAllocationConflictException { try { deployJob(host, job, Job.EMPTY_TOKEN); } catch (TokenVerificationException e) { throw Throwables.propagate(e); } }