@Test public void testTaskAssignmentManager() { TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); Map<String, String> expectedMap = ImmutableMap.of("Task0", "0", "Task1", "1", "Task2", "2", "Task3", "0", "Task4", "1"); for (Map.Entry<String, String> entry : expectedMap.entrySet()) { taskAssignmentManager.writeTaskContainerMapping(entry.getKey(), entry.getValue(), TaskMode.Active); } Map<String, String> localMap = taskAssignmentManager.readTaskAssignment(); assertEquals(expectedMap, localMap); taskAssignmentManager.close(); }
Map<String, String> taskToContainerId = taskAssignmentManager.readTaskAssignment(); taskToContainerId.values().forEach(id -> { try { taskAssignmentManager.deleteTaskContainerMappings(taskToContainerId.keySet()); return null;
TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); taskAssignmentManager.close();
@Test public void testTaskAssignmentManagerEmptyCoordinatorStream() { TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); Map<String, String> expectedMap = new HashMap<>(); Map<String, String> localMap = taskAssignmentManager.readTaskAssignment(); assertEquals(expectedMap, localMap); taskAssignmentManager.close(); } }
/** * Builds list of {@link Task} from job model in coordinator stream. * @param consumer system consumer associated with a job's coordinator stream. * @return list of {@link Task} constructed from job model in coordinator stream. */ protected List<Task> readTasksFromCoordinatorStream(CoordinatorStreamSystemConsumer consumer) { LocalityManager localityManager = new LocalityManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, Map<String, String>> containerIdToHostMapping = localityManager.readContainerLocality(); TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, String> taskNameToContainerIdMapping = taskAssignmentManager.readTaskAssignment(); StorageConfig storageConfig = new StorageConfig(consumer.getConfig()); List<String> storeNames = JavaConverters.seqAsJavaListConverter(storageConfig.getStoreNames()).asJava(); return taskNameToContainerIdMapping.entrySet() .stream() .map(entry -> { String hostName = containerIdToHostMapping.get(entry.getValue()).get(SetContainerHostMapping.HOST_KEY); return new Task(hostName, entry.getKey(), entry.getValue(), new ArrayList<>(), storeNames); }).collect(Collectors.toList()); } }
Mockito.doNothing().when(mockTaskAssignmentManager).writeTaskContainerMapping(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(mockTaskAssignmentManager).deleteTaskContainerMappings(Mockito.any()); Mockito.verify(mockTaskAssignmentManager).writeTaskContainerMapping("task-1", "test-container-id", TaskMode.Active); Mockito.verify(mockTaskAssignmentManager).writeTaskContainerMapping("task-2", "test-container-id", TaskMode.Active); Mockito.verify(mockTaskAssignmentManager).writeTaskContainerMapping("task-3", "test-container-id", TaskMode.Active); Mockito.verify(mockTaskAssignmentManager).writeTaskContainerMapping("task-4", "test-container-id", TaskMode.Active);
/** * Saves the task assignments specified by containers using the provided TaskAssignementManager. * * @param containers the set of containers from which the task assignments will be saved. * @param taskAssignmentManager the {@link TaskAssignmentManager} that will be used to save the mappings. */ private void saveTaskAssignments(Set<ContainerModel> containers, TaskAssignmentManager taskAssignmentManager) { for (ContainerModel container : containers) { for (TaskName taskName : container.getTasks().keySet()) { taskAssignmentManager.writeTaskContainerMapping(taskName.getTaskName(), container.getId()); } } }
@Test public void testGetGrouperMetadata() { // Mocking setup. LocalityManager mockLocalityManager = mock(LocalityManager.class); TaskAssignmentManager mockTaskAssignmentManager = Mockito.mock(TaskAssignmentManager.class); Map<String, Map<String, String>> localityMappings = new HashMap<>(); localityMappings.put("0", ImmutableMap.of(SetContainerHostMapping.HOST_KEY, "abc-affinity")); Map<String, String> taskAssignment = ImmutableMap.of("task-0", "0"); // Mock the container locality assignment. when(mockLocalityManager.readContainerLocality()).thenReturn(localityMappings); // Mock the container to task assignment. when(mockTaskAssignmentManager.readTaskAssignment()).thenReturn(taskAssignment); when(mockTaskAssignmentManager.readTaskModes()).thenReturn(Collections.singletonMap(new TaskName("task-0"), TaskMode.Active)); GrouperMetadataImpl grouperMetadata = JobModelManager.getGrouperMetadata(new MapConfig(), mockLocalityManager, mockTaskAssignmentManager); Mockito.verify(mockLocalityManager).readContainerLocality(); Mockito.verify(mockTaskAssignmentManager).readTaskAssignment(); Assert.assertEquals(ImmutableMap.of("0", new LocationId("abc-affinity"), "1", new LocationId("ANY_HOST")), grouperMetadata.getProcessorLocality()); Assert.assertEquals(ImmutableMap.of(new TaskName("task-0"), new LocationId("abc-affinity")), grouperMetadata.getTaskLocality()); }
@Before public void setup() throws Exception { when(mockStreamMetadataCache.getStreamMetadata(argThat(new ArgumentMatcher<scala.collection.immutable.Set<SystemStream>>() { @Override public boolean matches(Object argument) { scala.collection.immutable.Set<SystemStream> set = (scala.collection.immutable.Set<SystemStream>) argument; return set.equals(inputStreamSet); } }), anyBoolean())).thenReturn(mockStreamMetadataMap); when(mockStreamMetadata.getSystemStreamPartitionMetadata()).thenReturn(mockSspMetadataMap); PowerMockito.whenNew(TaskAssignmentManager.class).withAnyArguments().thenReturn(mockTaskManager); when(mockTaskManager.readTaskAssignment()).thenReturn(Collections.EMPTY_MAP); }
TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); taskAssignmentManager.close();
/** * Saves the task assignments specified by containers using the provided TaskAssignementManager. * * @param containers the set of containers from which the task assignments will be saved. * @param taskAssignmentManager the {@link TaskAssignmentManager} that will be used to save the mappings. */ private void saveTaskAssignments(Set<ContainerModel> containers, TaskAssignmentManager taskAssignmentManager) { for (ContainerModel container : containers) { for (TaskName taskName : container.getTasks().keySet()) { taskAssignmentManager.writeTaskContainerMapping(taskName.getTaskName(), container.getId()); } } }
@Test public void testDeleteMappings() { TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); Map<String, String> expectedMap = ImmutableMap.of("Task0", "0", "Task1", "1"); for (Map.Entry<String, String> entry : expectedMap.entrySet()) { taskAssignmentManager.writeTaskContainerMapping(entry.getKey(), entry.getValue(), TaskMode.Active); } Map<String, String> localMap = taskAssignmentManager.readTaskAssignment(); assertEquals(expectedMap, localMap); taskAssignmentManager.deleteTaskContainerMappings(localMap.keySet()); Map<String, String> deletedMap = taskAssignmentManager.readTaskAssignment(); assertTrue(deletedMap.isEmpty()); taskAssignmentManager.close(); }
TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); taskAssignmentManager.close();
Map<String, String> taskToContainerId = taskAssignmentManager.readTaskAssignment(); taskToContainerId.values().forEach(id -> { try { taskAssignmentManager.deleteTaskContainerMappings(taskToContainerId.keySet()); return null;
/** * Saves the task assignments specified by containers using the provided TaskAssignementManager. * * @param containers the set of containers from which the task assignments will be saved. * @param taskAssignmentManager the {@link TaskAssignmentManager} that will be used to save the mappings. */ private void saveTaskAssignments(Set<ContainerModel> containers, TaskAssignmentManager taskAssignmentManager) { for (ContainerModel container : containers) { for (TaskName taskName : container.getTasks().keySet()) { taskAssignmentManager.writeTaskContainerMapping(taskName.getTaskName(), container.getId()); } } }
TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); taskAssignmentManager.init(); try { List<TaskGroup> containers = getPreviousContainers(taskAssignmentManager, tasks.size()); taskAssignmentManager.close();
Map<String, String> taskToContainerId = taskAssignmentManager.readTaskAssignment(); taskToContainerId.values().forEach(id -> { try { taskAssignmentManager.deleteTaskContainerMappings(taskToContainerId.keySet()); return null;
/** * Saves the task assignments specified by containers using the provided TaskAssignementManager. * * @param containers the set of containers from which the task assignments will be saved. * @param taskAssignmentManager the {@link TaskAssignmentManager} that will be used to save the mappings. */ private void saveTaskAssignments(Set<ContainerModel> containers, TaskAssignmentManager taskAssignmentManager) { for (ContainerModel container : containers) { for (TaskName taskName : container.getTasks().keySet()) { taskAssignmentManager.writeTaskContainerMapping(taskName.getTaskName(), container.getId()); } } }
Map<String, String> taskToContainerId = taskAssignmentManager.readTaskAssignment(); taskToContainerId.values().forEach(id -> { try { taskAssignmentManager.deleteTaskContainerMappings(taskToContainerId.keySet()); return null;