public Set<ContainerModel> group(Set<TaskModel> taskModels, GrouperMetadata grouperMetadata) { if (this.standbyTasksEnabled) { return generateStandbyTasks(this.taskNameGrouper.group(taskModels, grouperMetadata), replicationFactor); } else { return this.taskNameGrouper.group(taskModels, grouperMetadata); } }
/** * Generate a container model map with standby tasks added and grouped into buddy containers. * Package-private for testing. * * @param containerModels The initial container model map. * @param replicationFactor The desired replication factor, if the replication-factor is n, we add n-1 standby tasks for each active task. * @return The generated map of containerModels with added containers, and the initial regular containers */ Set<ContainerModel> generateStandbyTasks(Set<ContainerModel> containerModels, int replicationFactor) { LOG.info("Received current containerModel map : {}, replicationFactor : {}", containerModels, replicationFactor); Set<ContainerModel> buddyContainers = new HashSet<>(); for (ContainerModel activeContainer : containerModels) { for (int replicaNum = 0; replicaNum < replicationFactor - 1; replicaNum++) { String buddyContainerId = getBuddyContainerId(activeContainer.getId(), replicaNum); ContainerModel buddyContainerModel = new ContainerModel(buddyContainerId, getTaskModelForBuddyContainer(activeContainer.getTasks(), replicaNum)); buddyContainers.add(buddyContainerModel); } } LOG.info("Adding buddy containers : {}", buddyContainers); buddyContainers.addAll(containerModels); return buddyContainers; }
private static Map<TaskName, TaskModel> getTaskModelForBuddyContainer( Map<TaskName, TaskModel> activeContainerTaskModel, int replicaNum) { Map<TaskName, TaskModel> standbyTaskModels = new HashMap<>(); for (TaskName taskName : activeContainerTaskModel.keySet()) { TaskName standbyTaskName = getStandbyTaskName(taskName, replicaNum); TaskModel standbyTaskModel = new TaskModel(standbyTaskName, activeContainerTaskModel.get(taskName).getSystemStreamPartitions(), activeContainerTaskModel.get(taskName).getChangelogPartition(), TaskMode.Standby); standbyTaskModels.put(standbyTaskName, standbyTaskModel); } LOG.info("Generated standbyTaskModels : {} for active task models : {}", standbyTaskModels, activeContainerTaskModel); return standbyTaskModels; }
@Test public void testBuddyContainerBasedGenerationIdentity() { this.standbyTaskGenerator = new TaskNameGrouperProxy(Mockito.mock(TaskNameGrouper.class), true, 2); Assert.assertEquals("Shouldnt add standby tasks to empty container map", Collections.emptySet(), this.standbyTaskGenerator.generateStandbyTasks(Collections.emptySet(), 1)); Assert.assertEquals("Shouldnt add standby tasks when repl factor = 1", getContainerMap(), this.standbyTaskGenerator.generateStandbyTasks(getContainerMap(), 1)); }
private void testBuddyContainerBasedGeneration(int replicationFactor) { this.standbyTaskGenerator = new TaskNameGrouperProxy(Mockito.mock(TaskNameGrouper.class), true, 2); standbyTaskGenerator.generateStandbyTasks(initialContainerModels, replicationFactor);
public Set<ContainerModel> group(Set<TaskModel> taskModels, List<String> containersIds) { if (this.standbyTasksEnabled) { return generateStandbyTasks(this.taskNameGrouper.group(taskModels, containersIds), replicationFactor); } else { return this.taskNameGrouper.group(taskModels, containersIds); } }