@Override public TaskNameGrouper build(Config config) { return new GroupByContainerCount(config); } }
validateTasks(tasks); return group(tasks); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); if (containers == null || containers.size() == 1 || containerCount == 1) { log.info("Balancing does not apply. Invoking grouper."); Set<ContainerModel> models = group(tasks); saveTaskAssignments(models, taskAssignmentManager); return models; if (containerDelta == 0) { log.info("Container count has not changed. Reusing previous container models."); return buildContainerModels(tasks, containers); int[] expectedTaskCountPerContainer = calculateTaskCountPerContainer(tasks.size(), prevContainerCount, containerCount); List<TaskGroup> newContainers = createContainers(prevContainerCount, containerCount); containers.addAll(newContainers); } else { containers = containers.subList(0, containerCount); assignTasksToContainers(expectedTaskCountPerContainer, taskNamesToReassign, containers); Set<ContainerModel> models = buildContainerModels(tasks, containers); saveTaskAssignments(models, taskAssignmentManager);
/** * Reads the task-container mapping from the provided {@link GrouperMetadata} and returns a * list of TaskGroups, ordered ascending by containerId. * * @param grouperMetadata the {@link GrouperMetadata} will be used to retrieve the previous task to container assignments. * @param taskCount the number of tasks, for validation against the persisted tasks. * @return a list of TaskGroups, ordered ascending by containerId or {@code null} * if the previous mapping doesn't exist or isn't usable. */ private List<TaskGroup> getPreviousContainers(GrouperMetadata grouperMetadata, int taskCount) { Map<TaskName, String> taskToContainerId = grouperMetadata.getPreviousTaskToProcessorAssignment(); if (taskToContainerId.isEmpty()) { LOG.info("No task assignment map was saved."); return null; } else if (taskCount != taskToContainerId.size()) { return null; } List<TaskGroup> containers; try { containers = getOrderedContainers(taskToContainerId); } catch (Exception e) { LOG.error("Exception while parsing task mapping", e); return null; } return containers; }
validateTasks(tasks); List<TaskGroup> containers = getPreviousContainers(grouperMetadata, tasks.size()); if (containers == null || containers.size() == 1 || containerCount == 1) { LOG.info("Balancing does not apply. Invoking grouper."); return group(tasks); int[] expectedTaskCountPerContainer = calculateTaskCountPerContainer(tasks.size(), prevContainerCount, containerCount); List<TaskGroup> newContainers = createContainers(prevContainerCount, containerCount); containers.addAll(newContainers); } else { assignTasksToContainers(expectedTaskCountPerContainer, taskNamesToReassign, containers);
@Test(expected = IllegalArgumentException.class) public void testGroupEmptyTasks() { new GroupByContainerCount(1).group(new HashSet<>()); }
@Test public void testBalancerWithNullLocalityManager() { Set<TaskModel> taskModels = generateTaskModels(3); Set<ContainerModel> groupContainers = new GroupByContainerCount(3).group(taskModels); Set<ContainerModel> balanceContainers = new GroupByContainerCount(3).balance(taskModels, null); // Results should be the same as calling group() assertEquals(groupContainers, balanceContainers); } }
/** * {@inheritDoc} */ @Override public Set<ContainerModel> group(Set<TaskModel> tasks) { validateTasks(tasks); // Sort tasks by taskName. List<TaskModel> sortedTasks = new ArrayList<>(tasks); Collections.sort(sortedTasks); // Map every task to a container in round-robin fashion. Map<TaskName, TaskModel>[] taskGroups = new Map[containerCount]; for (int i = 0; i < containerCount; i++) { taskGroups[i] = new HashMap<>(); } for (int i = 0; i < sortedTasks.size(); i++) { TaskModel tm = sortedTasks.get(i); taskGroups[i % containerCount].put(tm.getTaskName(), tm); } // Convert to a Set of ContainerModel Set<ContainerModel> containerModels = new HashSet<>(); for (int i = 0; i < containerCount; i++) { containerModels.add(new ContainerModel(String.valueOf(i), taskGroups[i])); } return Collections.unmodifiableSet(containerModels); }
@Test public void testGroupTaskCountIncrease() { int taskCount = 3; Set<TaskModel> taskModels = generateTaskModels(taskCount); Set<ContainerModel> prevContainers = new GroupByContainerCount(2).group(generateTaskModels(taskCount - 1)); // Here's the key step Map<TaskName, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers); GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping); Set<ContainerModel> groupContainers = new GroupByContainerCount(1).group(taskModels); Set<ContainerModel> balanceContainers = new GroupByContainerCount(1).group(taskModels, grouperMetadata); // Results should be the same as calling group() assertEquals(groupContainers, balanceContainers); }
@Override public Set<ContainerModel> group(Set<TaskModel> tasks) { validateTasks(tasks); // Sort tasks by taskName. List<TaskModel> sortedTasks = new ArrayList<>(tasks); Collections.sort(sortedTasks); // Map every task to a container in round-robin fashion. Map<TaskName, TaskModel>[] taskGroups = new Map[containerCount]; for (int i = 0; i < containerCount; i++) { taskGroups[i] = new HashMap<>(); } for (int i = 0; i < sortedTasks.size(); i++) { TaskModel tm = sortedTasks.get(i); taskGroups[i % containerCount].put(tm.getTaskName(), tm); } // Convert to a Set of ContainerModel Set<ContainerModel> containerModels = new HashSet<>(); for (int i = 0; i < containerCount; i++) { containerModels.add(new ContainerModel(String.valueOf(i), taskGroups[i])); } return Collections.unmodifiableSet(containerModels); }
validateTasks(tasks); return group(tasks); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); if (containers == null || containers.size() == 1 || containerCount == 1) { log.info("Balancing does not apply. Invoking grouper."); Set<ContainerModel> models = group(tasks); saveTaskAssignments(models, taskAssignmentManager); return models; if (containerDelta == 0) { log.info("Container count has not changed. Reusing previous container models."); return buildContainerModels(tasks, containers); int[] expectedTaskCountPerContainer = calculateTaskCountPerContainer(tasks.size(), prevContainerCount, containerCount); List<TaskGroup> newContainers = createContainers(prevContainerCount, containerCount); containers.addAll(newContainers); } else { containers = containers.subList(0, containerCount); assignTasksToContainers(expectedTaskCountPerContainer, taskNamesToReassign, containers); Set<ContainerModel> models = buildContainerModels(tasks, containers); saveTaskAssignments(models, taskAssignmentManager);
@Test public void testBalancerNewContainerCountOne() { Set<TaskModel> taskModels = generateTaskModels(3); Set<ContainerModel> prevContainers = new GroupByContainerCount(3).group(taskModels); Map<TaskName, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers); GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping); Set<ContainerModel> groupContainers = new GroupByContainerCount(1).group(taskModels); Set<ContainerModel> balanceContainers = new GroupByContainerCount(1).group(taskModels, grouperMetadata); // Results should be the same as calling group() assertEquals(groupContainers, balanceContainers); }
@Override public TaskNameGrouper build(Config config) { return new GroupByContainerCount(config); } }
@Override public Set<ContainerModel> group(Set<TaskModel> tasks) { validateTasks(tasks); // Sort tasks by taskName. List<TaskModel> sortedTasks = new ArrayList<>(tasks); Collections.sort(sortedTasks); // Map every task to a container in round-robin fashion. Map<TaskName, TaskModel>[] taskGroups = new Map[containerCount]; for (int i = 0; i < containerCount; i++) { taskGroups[i] = new HashMap<>(); } for (int i = 0; i < sortedTasks.size(); i++) { TaskModel tm = sortedTasks.get(i); taskGroups[i % containerCount].put(tm.getTaskName(), tm); } // Convert to a Set of ContainerModel Set<ContainerModel> containerModels = new HashSet<>(); for (int i = 0; i < containerCount; i++) { containerModels.add(new ContainerModel(String.valueOf(i), taskGroups[i])); } return Collections.unmodifiableSet(containerModels); }
containers = getOrderedContainers(taskToContainerId); } catch (Exception e) { log.error("Exception while parsing task mapping", e);
validateTasks(tasks); return group(tasks); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); if (containers == null || containers.size() == 1 || containerCount == 1) { log.info("Balancing does not apply. Invoking grouper."); Set<ContainerModel> models = group(tasks); saveTaskAssignments(models, taskAssignmentManager); return models; if (containerDelta == 0) { log.info("Container count has not changed. Reusing previous container models."); return buildContainerModels(tasks, containers); int[] expectedTaskCountPerContainer = calculateTaskCountPerContainer(tasks.size(), prevContainerCount, containerCount); List<TaskGroup> newContainers = createContainers(prevContainerCount, containerCount); containers.addAll(newContainers); } else { containers = containers.subList(0, containerCount); assignTasksToContainers(expectedTaskCountPerContainer, taskNamesToReassign, containers); Set<ContainerModel> models = buildContainerModels(tasks, containers); saveTaskAssignments(models, taskAssignmentManager);
@Test public void testBalancerOldContainerCountOne() { Set<TaskModel> taskModels = generateTaskModels(3); Set<ContainerModel> prevContainers = new GroupByContainerCount(1).group(taskModels); Map<TaskName, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers); GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), new HashMap<>(), prevTaskToContainerMapping); Set<ContainerModel> groupContainers = new GroupByContainerCount(3).group(taskModels); Set<ContainerModel> balanceContainers = new GroupByContainerCount(3).group(taskModels, grouperMetadata); // Results should be the same as calling group() assertEquals(groupContainers, balanceContainers); }
@Override public TaskNameGrouper build(Config config) { return new GroupByContainerCount(config); } }
@Override public Set<ContainerModel> group(Set<TaskModel> tasks) { validateTasks(tasks); // Sort tasks by taskName. List<TaskModel> sortedTasks = new ArrayList<>(tasks); Collections.sort(sortedTasks); // Map every task to a container in round-robin fashion. Map<TaskName, TaskModel>[] taskGroups = new Map[containerCount]; for (int i = 0; i < containerCount; i++) { taskGroups[i] = new HashMap<>(); } for (int i = 0; i < sortedTasks.size(); i++) { TaskModel tm = sortedTasks.get(i); taskGroups[i % containerCount].put(tm.getTaskName(), tm); } // Convert to a Set of ContainerModel Set<ContainerModel> containerModels = new HashSet<>(); for (int i = 0; i < containerCount; i++) { containerModels.add(new ContainerModel(String.valueOf(i), taskGroups[i])); } return Collections.unmodifiableSet(containerModels); }
containers = getOrderedContainers(taskToContainerId); } catch (Exception e) { log.error("Exception while parsing task mapping", e);
validateTasks(tasks); return group(tasks); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); if (containers == null || containers.size() == 1 || containerCount == 1) { log.info("Balancing does not apply. Invoking grouper."); Set<ContainerModel> models = group(tasks); saveTaskAssignments(models, taskAssignmentManager); return models; if (containerDelta == 0) { log.info("Container count has not changed. Reusing previous container models."); return buildContainerModels(tasks, containers); int[] expectedTaskCountPerContainer = calculateTaskCountPerContainer(tasks.size(), prevContainerCount, containerCount); List<TaskGroup> newContainers = createContainers(prevContainerCount, containerCount); containers.addAll(newContainers); } else { containers = containers.subList(0, containerCount); assignTasksToContainers(expectedTaskCountPerContainer, taskNamesToReassign, containers); Set<ContainerModel> models = buildContainerModels(tasks, containers); saveTaskAssignments(models, taskAssignmentManager);