public LocationId getLocationId() { return new LocationId(locationId); }
public LocationId getLocationId() { return new LocationId(locationId); }
public LocationId getLocationId() { return new LocationId(locationId); }
public LocationId getLocationId() { return new LocationId(locationId); }
public LocationId getLocationId() { return new LocationId(locationId); }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
public Map<TaskName, LocationId> readTaskLocality() { Map<TaskName, LocationId> taskLocality = new HashMap<>(); String taskLocalityPath = keyBuilder.getTaskLocalityPath(); List<String> tasks = new ArrayList<>(); if (zkClient.exists(taskLocalityPath)) { tasks = zkClient.getChildren(taskLocalityPath); } for (String taskName : tasks) { String taskPath = String.format("%s/%s", keyBuilder.getTaskLocalityPath(), taskName); String locationId = zkClient.readData(taskPath, true); if (locationId != null) { taskLocality.put(new TaskName(taskName), new LocationId(locationId)); } } return taskLocality; }
@Test public void testGetProcessorLocality() { // Mock the dependencies. LocalityManager mockLocalityManager = mock(LocalityManager.class); Map<String, Map<String, String>> localityMappings = new HashMap<>(); localityMappings.put("0", ImmutableMap.of(SetContainerHostMapping.HOST_KEY, "abc-affinity")); // Mock the container locality assignment. when(mockLocalityManager.readContainerLocality()).thenReturn(localityMappings); Map<String, LocationId> processorLocality = JobModelManager.getProcessorLocality(new MapConfig(), mockLocalityManager); Mockito.verify(mockLocalityManager).readContainerLocality(); Assert.assertEquals(ImmutableMap.of("0", new LocationId("abc-affinity"), "1", new LocationId("ANY_HOST")), processorLocality); }
@Test public void testReadTaskLocalityShouldReturnAllTheExistingLocalityValue() { zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-1")); zkUtils.writeTaskLocality(new TaskName("task-2"), new LocationId("LocationId-2")); zkUtils.writeTaskLocality(new TaskName("task-3"), new LocationId("LocationId-3")); zkUtils.writeTaskLocality(new TaskName("task-4"), new LocationId("LocationId-4")); zkUtils.writeTaskLocality(new TaskName("task-5"), new LocationId("LocationId-5")); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-1"), new TaskName("task-2"), new LocationId("LocationId-2"), new TaskName("task-3"), new LocationId("LocationId-3"), new TaskName("task-4"), new LocationId("LocationId-4"), new TaskName("task-5"), new LocationId("LocationId-5")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); }
@Test public void testReadAfterWriteTaskLocality() { zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-1")); zkUtils.writeTaskLocality(new TaskName("task-2"), new LocationId("LocationId-2")); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-1"), new TaskName("task-2"), new LocationId("LocationId-2")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); }
@Test public void testWriteTaskLocalityShouldUpdateTheExistingValue() { zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-1")); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-1")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-2")); taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-2")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); }
@Test public void testGetGrouperMetadata() { // Mocking setup. LocalityManager mockLocalityManager = mock(LocalityManager.class); TaskAssignmentManager mockTaskAssignmentManager = Mockito.mock(TaskAssignmentManager.class); Map<String, Map<String, String>> localityMappings = new HashMap<>(); localityMappings.put("0", ImmutableMap.of(SetContainerHostMapping.HOST_KEY, "abc-affinity")); Map<String, String> taskAssignment = ImmutableMap.of("task-0", "0"); // Mock the container locality assignment. when(mockLocalityManager.readContainerLocality()).thenReturn(localityMappings); // Mock the container to task assignment. when(mockTaskAssignmentManager.readTaskAssignment()).thenReturn(taskAssignment); when(mockTaskAssignmentManager.readTaskModes()).thenReturn(Collections.singletonMap(new TaskName("task-0"), TaskMode.Active)); GrouperMetadataImpl grouperMetadata = JobModelManager.getGrouperMetadata(new MapConfig(), mockLocalityManager, mockTaskAssignmentManager); Mockito.verify(mockLocalityManager).readContainerLocality(); Mockito.verify(mockTaskAssignmentManager).readTaskAssignment(); Assert.assertEquals(ImmutableMap.of("0", new LocationId("abc-affinity"), "1", new LocationId("ANY_HOST")), grouperMetadata.getProcessorLocality()); Assert.assertEquals(ImmutableMap.of(new TaskName("task-0"), new LocationId("abc-affinity")), grouperMetadata.getTaskLocality()); }
@Test public void testGenerateContainerModelForSingleContainer() { TaskNameGrouper taskNameGrouper = buildSimpleGrouper(1); String testProcessorId1 = "testProcessorId1"; LocationId testLocationId1 = new LocationId("testLocationId1"); LocationId testLocationId2 = new LocationId("testLocationId2"); LocationId testLocationId3 = new LocationId("testLocationId3"); TaskName testTaskName1 = new TaskName("testTasKId1"); TaskName testTaskName2 = new TaskName("testTaskId2"); TaskName testTaskName3 = new TaskName("testTaskId3"); TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0)); TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1)); TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2)); Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1, testTaskName2, testLocationId2, testTaskName3, testLocationId3); GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>()); Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3); Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1, testTaskName2, testTaskModel2, testTaskName3, testTaskModel3))); Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata); assertEquals(expectedContainerModels, actualContainerModels); }
@Test public void testShouldGenerateCorrectContainerModelWhenTaskLocalityIsEmpty() { TaskNameGrouper taskNameGrouper = buildSimpleGrouper(3); String testProcessorId1 = "testProcessorId1"; String testProcessorId2 = "testProcessorId2"; String testProcessorId3 = "testProcessorId3"; LocationId testLocationId1 = new LocationId("testLocationId1"); LocationId testLocationId2 = new LocationId("testLocationId2"); LocationId testLocationId3 = new LocationId("testLocationId3"); TaskName testTaskName1 = new TaskName("testTasKId1"); TaskName testTaskName2 = new TaskName("testTaskId2"); TaskName testTaskName3 = new TaskName("testTaskId3"); TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0)); TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1)); TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2)); Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1, testProcessorId2, testLocationId2, testProcessorId3, testLocationId3); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1); GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>()); Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3); Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1)), new ContainerModel(testProcessorId2, ImmutableMap.of(testTaskName2, testTaskModel2)), new ContainerModel(testProcessorId3, ImmutableMap.of(testTaskName3, testTaskModel3))); Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata); assertEquals(expectedContainerModels, actualContainerModels); }
@Test public void testLocalityMapWithHostAffinity() { Config config = new MapConfig(new HashMap<String, String>() { { put("cluster-manager.container.count", "1"); put("cluster-manager.container.memory.mb", "512"); put("cluster-manager.container.retry.count", "1"); put("cluster-manager.container.retry.window.ms", "1999999999"); put("cluster-manager.allocator.sleep.ms", "10"); put("yarn.package.path", "/foo"); put("task.inputs", "test-system.test-stream"); put("systems.test-system.samza.factory", "org.apache.samza.system.MockSystemFactory"); put("systems.test-system.samza.key.serde", "org.apache.samza.serializers.JsonSerde"); put("systems.test-system.samza.msg.serde", "org.apache.samza.serializers.JsonSerde"); put("job.host-affinity.enabled", "true"); } }); LocalityManager mockLocalityManager = mock(LocalityManager.class); localityMappings.put("0", new HashMap<String, String>() { { put(SetContainerHostMapping.HOST_KEY, "abc-affinity"); } }); when(mockLocalityManager.readContainerLocality()).thenReturn(this.localityMappings); Map<String, LocationId> containerLocality = ImmutableMap.of("0", new LocationId("abc-affinity")); this.jobModelManager = JobModelManagerTestUtil.getJobModelManagerUsingReadModel(config, mockStreamMetadataCache, server, mockLocalityManager, containerLocality); assertEquals(jobModelManager.jobModel().getAllContainerLocality(), new HashMap<String, String>() { { this.put("0", "abc-affinity"); } }); }
@Test public void testLocalityMapWithoutHostAffinity() { Config config = new MapConfig(new HashMap<String, String>() { { put("cluster-manager.container.count", "1"); put("cluster-manager.container.memory.mb", "512"); put("cluster-manager.container.retry.count", "1"); put("cluster-manager.container.retry.window.ms", "1999999999"); put("cluster-manager.allocator.sleep.ms", "10"); put("yarn.package.path", "/foo"); put("task.inputs", "test-system.test-stream"); put("systems.test-system.samza.factory", "org.apache.samza.system.MockSystemFactory"); put("systems.test-system.samza.key.serde", "org.apache.samza.serializers.JsonSerde"); put("systems.test-system.samza.msg.serde", "org.apache.samza.serializers.JsonSerde"); put("job.host-affinity.enabled", "false"); } }); LocalityManager mockLocalityManager = mock(LocalityManager.class); localityMappings.put("0", new HashMap<String, String>() { { put(SetContainerHostMapping.HOST_KEY, "abc-affinity"); } }); when(mockLocalityManager.readContainerLocality()).thenReturn(new HashMap<>()); Map<String, LocationId> containerLocality = ImmutableMap.of("0", new LocationId("abc-affinity")); this.jobModelManager = JobModelManagerTestUtil.getJobModelManagerUsingReadModel(config, mockStreamMetadataCache, server, mockLocalityManager, containerLocality); assertEquals(jobModelManager.jobModel().getAllContainerLocality(), new HashMap<String, String>() { { this.put("0", null); } }); }