/** * Create a plan of tasks to be executed in order to perform the cluster operation. Each item in the list represents * a stage of tasks that can be performed. All tasks in a stage may be run in parallel, but every task in a stage * must be successfully completed before moving on to the next stage. * * @return Plan of tasks to be executed in order to perform a cluster operation. */ public List<Set<TaskNode>> linearizeDependentTasks() { TaskDag taskDag = createTaskDag(); long start = System.currentTimeMillis(); List<Set<TaskNode>> linearizedTasks = taskDag.linearize(); long dur = System.currentTimeMillis() - start; LOG.debug("took {} ms to linearize action plan.", dur); return linearizedTasks; }
/** * Creates a DAG (directed acyclic graph) of tasks to execute in order to perform the cluster job. * * @return Task dag for the cluster operation. */ TaskDag createTaskDag() { long start = System.currentTimeMillis(); TaskDag taskDag = new TaskDag(); List<ProvisionerAction> actionOrder = actions.getActionOrder().get(clusterAction); for (Node node : nodeMap.values()) { if (!shouldPlanNode(node.getId())) { continue; } for (Service service : node.getServices()) { if (!shouldPlanService(service.getName())) { continue; } addDependencies(taskDag, actionOrder, service, node); } } long dur = System.currentTimeMillis() - start; LOG.debug("took {} ms to create action plan.", dur); return taskDag; }
/** * Makes sure that not more than one task per host is present in a stage. If task t1 and t2 for the same host are * present in stage i, then it create a new stage i.5 and adds either t1 or t2 to i.5. * Task i.5 will execute after i, but before i+1, so as to satisfy all dependencies. * @param tasks Initial task list, where there may be multiple tasks on same host in any stage. * @return the deduped task list. */ static List<Set<ClusterTask>> deDupNodePerStage(List<Set<ClusterTask>> tasks) { // Map contains nodeId and task. List<Map<String, ClusterTask>> deDupTasks = Lists.newArrayList(); for (Set<ClusterTask> stage : tasks) { List<Map<String, ClusterTask>> newStages = Lists.newArrayList(); for (ClusterTask task : stage) { if (!addToExistingStage(task, newStages)) { Map<String, ClusterTask> newStage = Maps.newHashMap(); newStage.put(task.getNodeId(), task); newStages.add(newStage); } } deDupTasks.addAll(newStages); } return Lists.newArrayList(Iterables.transform(deDupTasks, MAP_CLUSTER_TASK_FUNCTION)); }
private Set<String> expandServices(Set<String> services, ClusterAction action) { switch (action) { case START_SERVICES: return expandStartServices(services); case STOP_SERVICES: case RESTART_SERVICES: return expandStopServices(services); default: return services; } }
@Test public void testNoEdgeNodesInDag() { Service s1 = Service.builder().setName("s1").build(); Node node1 = new Node("node1", "1", ImmutableSet.<Service>of(s1), NodeProperties.builder().setHostname("host1").addIPAddress("access_v4", "ip1").build()); Node node2 = new Node("node2", "1", ImmutableSet.<Service>of(s1), NodeProperties.builder().setHostname("host2").addIPAddress("access_v4", "ip2").build()); Set<Node> clusterNodes = ImmutableSet.of(node1, node2); TaskDag expected = new TaskDag(); TaskNode taskNode1 = new TaskNode(node1.getId(), ProvisionerAction.DELETE.name(), ""); TaskNode taskNode2 = new TaskNode(node2.getId(), ProvisionerAction.DELETE.name(), ""); expected.addTaskNode(taskNode1); expected.addTaskNode(taskNode2); ClusterJob job = new ClusterJob(JobId.fromString("123-001"), ClusterAction.CLUSTER_DELETE); JobPlanner planner = new JobPlanner(job, clusterNodes); TaskDag actual = planner.createTaskDag(); Assert.assertEquals(expected, actual); List<Set<TaskNode>> linearizedTasks = actual.linearize(); Assert.assertEquals(1, linearizedTasks.size()); Assert.assertEquals(2, linearizedTasks.get(0).size()); Assert.assertTrue(linearizedTasks.get(0).contains(taskNode1)); Assert.assertTrue(linearizedTasks.get(0).contains(taskNode2)); }
JobPlanner jobPlanner = new JobPlanner(job, clusterNodes); List<Set<TaskNode>> linearizedTasks = jobPlanner.linearizeDependentTasks(); jobPlanner.getServiceMap(), clusterAction, jobPlanner.getNodeMap()); clusterTasks = JobPlanner.deDupNodePerStage(clusterTasks);
@Test public void testDedupNodesPerStage() throws Exception { List<Set<ClusterTask>> tasks = ImmutableList.of( createSortedSet(createClusterTask("INSTALL", "1-1-1", "host1"), createClusterTask("CONFIGURE", "1-1-2", "host1"), createClusterTask("INSTALL", "1-1-3", "host3")), createSortedSet(createClusterTask("INSTALL", "1-1-12", "host1"), createClusterTask("INSTALL", "1-1-22", "host2"), createClusterTask("INSTALL", "1-1-32", "host3")), createSortedSet(createClusterTask("INSTALL", "1-1-13", "host1"), createClusterTask("CONFIGURE", "1-1-22", "host1"), createClusterTask("INSTALL", "1-1-32", "host3")) ); List<Set<ClusterTask>> actual = JobPlanner.deDupNodePerStage(tasks); List<Set<ClusterTask>> expected = ImmutableList.of( createSortedSet(createClusterTask("INSTALL", "1-1-1", "host1"), createClusterTask("INSTALL", "1-1-3", "host3")), createSortedSet(createClusterTask("CONFIGURE", "1-1-2", "host1")), createSortedSet(createClusterTask("INSTALL", "1-1-12", "host1"), createClusterTask("INSTALL", "1-1-22", "host2"), createClusterTask("INSTALL", "1-1-32", "host3")), createSortedSet(createClusterTask("INSTALL", "1-1-13", "host1"), createClusterTask("INSTALL", "1-1-32", "host3")), createSortedSet(createClusterTask("CONFIGURE", "1-1-22", "host1")) ); Assert.assertEquals(expected, actual); }
public JobPlanner(ClusterJob job, Set<Node> clusterNodes) { this.clusterAction = job.getClusterAction(); this.nodesToPlan = job.getPlannedNodes(); this.serviceNodeMap = ArrayListMultimap.create(); this.serviceMap = Maps.newHashMap(); this.nodeMap = Maps.newHashMap(); for (Node node : clusterNodes) { for (Service service : node.getServices()) { serviceNodeMap.put(service.getName(), node); serviceMap.put(service.getName(), service); } nodeMap.put(node.getId(), node); } this.dependencyResolver = new ServiceDependencyResolver(actions, serviceMap); if (job.getPlannedServices() != null) { this.servicesToPlan = ImmutableSet.copyOf(expandServices(job.getPlannedServices(), clusterAction)); } else { this.servicesToPlan = null; } }
JobPlanner planner = new JobPlanner(job, clusterNodes); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); TaskDag expected = new TaskDag(); new TaskNode(node1.getId(), ProvisionerAction.START.name(), "s3")); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node2.getId(), ProvisionerAction.STOP.name(), s2.getName()), new TaskNode(node1.getId(), ProvisionerAction.STOP.name(), s1.getName())); Assert.assertEquals(expected, planner.createTaskDag()); planner = new JobPlanner(job, clusterNodes); Assert.assertEquals(expected, planner.createTaskDag()); planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node2.getId(), ProvisionerAction.STOP.name(), s2.getName()), new TaskNode(node1.getId(), ProvisionerAction.STOP.name(), s1.getName())); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node1.getId(), ProvisionerAction.START.name(), s1.getName()), new TaskNode(node2.getId(), ProvisionerAction.START.name(), s2.getName())); Assert.assertEquals(expected, planner.createTaskDag()); planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node1.getId(), ProvisionerAction.START.name(), s1.getName()), new TaskNode(node2.getId(), ProvisionerAction.START.name(), s2.getName())); Assert.assertEquals(expected, planner.createTaskDag()); planner = new JobPlanner(job, clusterNodes); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node1.getId(), ProvisionerAction.CONFIGURE.name(), "s1"), new TaskNode(node1.getId(), ProvisionerAction.START.name(), "s1")); Assert.assertEquals(expected, planner.createTaskDag()); planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node1.getId(), ProvisionerAction.START.name(), "s1"), new TaskNode(node2.getId(), ProvisionerAction.START.name(), "s3")); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node2.getId(), ProvisionerAction.STOP.name(), s3.getName()), new TaskNode(node2.getId(), ProvisionerAction.START.name(), s3.getName())); Assert.assertEquals(expected, planner.createTaskDag()); planner = new JobPlanner(job, clusterNodes); expected.addDependency(new TaskNode(node2.getId(), ProvisionerAction.STOP.name(), s3.getName()), new TaskNode(node2.getId(), ProvisionerAction.START.name(), s3.getName())); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); TaskDag expected = new TaskDag(); new TaskNode(node2.getId(), ProvisionerAction.START.name(), "s1")); Assert.assertEquals(expected, planner.createTaskDag()); job = new ClusterJob(JobId.fromString("123-002"), ClusterAction.ADD_SERVICES, ImmutableSet.<String>of(s2.getName(), s3.getName()), ImmutableSet.<String>of(node2.getId())); planner = new JobPlanner(job, clusterNodes); expected = new TaskDag(); new TaskNode(node2.getId(), ProvisionerAction.START.name(), "s2")); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); TaskDag expected = new TaskDag(); new TaskNode(node1.getId(), ProvisionerAction.START.name(), "s3")); Assert.assertEquals(expected, planner.createTaskDag());
JobPlanner planner = new JobPlanner(job, clusterNodes); TaskDag dag = planner.createTaskDag();
JobPlanner planner = new JobPlanner(job, clusterNodes); Assert.assertEquals(expected, planner.createTaskDag());