private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
/** * Method to write locality information to the {@link MetadataStore}. This method is used in {@link SamzaContainer}. * * @param containerId the {@link SamzaContainer} ID * @param hostName the hostname */ public void writeContainerToHostMapping(String containerId, String hostName) { Map<String, Map<String, String>> containerToHostMapping = readContainerLocality(); Map<String, String> existingMappings = containerToHostMapping.get(containerId); String existingHostMapping = existingMappings != null ? existingMappings.get(SetContainerHostMapping.HOST_KEY) : null; if (existingHostMapping != null && !existingHostMapping.equals(hostName)) { LOG.info("Container {} moved from {} to {}", new Object[]{containerId, existingHostMapping, hostName}); } else { LOG.info("Container {} started at {}", containerId, hostName); } metadataStore.put(containerId, valueSerde.toBytes(hostName)); }
/** * Method to write locality information to the {@link MetadataStore}. This method is used in {@link SamzaContainer}. * * @param containerId the {@link SamzaContainer} ID * @param hostName the hostname */ public void writeContainerToHostMapping(String containerId, String hostName) { Map<String, Map<String, String>> containerToHostMapping = readContainerLocality(); Map<String, String> existingMappings = containerToHostMapping.get(containerId); String existingHostMapping = existingMappings != null ? existingMappings.get(SetContainerHostMapping.HOST_KEY) : null; if (existingHostMapping != null && !existingHostMapping.equals(hostName)) { LOG.info("Container {} moved from {} to {}", new Object[]{containerId, existingHostMapping, hostName}); } else { LOG.info("Container {} started at {}", containerId, hostName); } metadataStore.put(keySerde.toBytes(containerId), valueSerde.toBytes(hostName)); }
/** * Method to write locality information to the {@link MetadataStore}. This method is used in {@link SamzaContainer}. * * @param containerId the {@link SamzaContainer} ID * @param hostName the hostname */ public void writeContainerToHostMapping(String containerId, String hostName) { Map<String, Map<String, String>> containerToHostMapping = readContainerLocality(); Map<String, String> existingMappings = containerToHostMapping.get(containerId); String existingHostMapping = existingMappings != null ? existingMappings.get(SetContainerHostMapping.HOST_KEY) : null; if (existingHostMapping != null && !existingHostMapping.equals(hostName)) { LOG.info("Container {} moved from {} to {}", new Object[]{containerId, existingHostMapping, hostName}); } else { LOG.info("Container {} started at {}", containerId, hostName); } metadataStore.put(keySerde.toBytes(containerId), valueSerde.toBytes(hostName)); }
/** * Method to write locality information to the {@link MetadataStore}. This method is used in {@link SamzaContainer}. * * @param containerId the {@link SamzaContainer} ID * @param hostName the hostname */ public void writeContainerToHostMapping(String containerId, String hostName) { Map<String, Map<String, String>> containerToHostMapping = readContainerLocality(); Map<String, String> existingMappings = containerToHostMapping.get(containerId); String existingHostMapping = existingMappings != null ? existingMappings.get(SetContainerHostMapping.HOST_KEY) : null; if (existingHostMapping != null && !existingHostMapping.equals(hostName)) { LOG.info("Container {} moved from {} to {}", new Object[]{containerId, existingHostMapping, hostName}); } else { LOG.info("Container {} started at {}", containerId, hostName); } metadataStore.put(keySerde.toBytes(containerId), valueSerde.toBytes(hostName)); }
/** * Method to write locality information to the {@link MetadataStore}. This method is used in {@link SamzaContainer}. * * @param containerId the {@link SamzaContainer} ID * @param hostName the hostname */ public void writeContainerToHostMapping(String containerId, String hostName) { Map<String, Map<String, String>> containerToHostMapping = readContainerLocality(); Map<String, String> existingMappings = containerToHostMapping.get(containerId); String existingHostMapping = existingMappings != null ? existingMappings.get(SetContainerHostMapping.HOST_KEY) : null; if (existingHostMapping != null && !existingHostMapping.equals(hostName)) { LOG.info("Container {} moved from {} to {}", new Object[]{containerId, existingHostMapping, hostName}); } else { LOG.info("Container {} started at {}", containerId, hostName); } metadataStore.put(keySerde.toBytes(containerId), valueSerde.toBytes(hostName)); }
private JobModelManager getJobModelManagerWithHostAffinity(Map<String, String> containerIdToHost) { Map<String, Map<String, String>> localityMap = new HashMap<>(); containerIdToHost.forEach((containerId, host) -> { localityMap.put(containerId, ImmutableMap.of(SetContainerHostMapping.HOST_KEY, containerIdToHost.get(containerId))); }); LocalityManager mockLocalityManager = mock(LocalityManager.class); when(mockLocalityManager.readContainerLocality()).thenReturn(localityMap); return JobModelManagerTestUtil.getJobModelManagerWithLocalityManager(getConfig(), containerIdToHost.size(), mockLocalityManager, this.server); }
private JobModelManager initializeJobModelManager(Config config, int containerCount) { Map<String, Map<String, String>> localityMap = new HashMap<>(); localityMap.put("0", new HashMap<String, String>() { { put(SetContainerHostMapping.HOST_KEY, "abc"); } }); LocalityManager mockLocalityManager = mock(LocalityManager.class); when(mockLocalityManager.readContainerLocality()).thenReturn(localityMap); return JobModelManagerTestUtil.getJobModelManagerWithLocalityManager(getConfig(), containerCount, mockLocalityManager, new MockHttpServer("/", 7777, null, new ServletHolder(DefaultServlet.class))); }
@Test public void testGetProcessorLocality() { // Mock the dependencies. LocalityManager mockLocalityManager = mock(LocalityManager.class); Map<String, Map<String, String>> localityMappings = new HashMap<>(); localityMappings.put("0", ImmutableMap.of(SetContainerHostMapping.HOST_KEY, "abc-affinity")); // Mock the container locality assignment. when(mockLocalityManager.readContainerLocality()).thenReturn(localityMappings); Map<String, LocationId> processorLocality = JobModelManager.getProcessorLocality(new MapConfig(), mockLocalityManager); Mockito.verify(mockLocalityManager).readContainerLocality(); Assert.assertEquals(ImmutableMap.of("0", new LocationId("abc-affinity"), "1", new LocationId("ANY_HOST")), processorLocality); }
@Test public void testGetGrouperMetadata() { // Mocking setup. LocalityManager mockLocalityManager = mock(LocalityManager.class); TaskAssignmentManager mockTaskAssignmentManager = Mockito.mock(TaskAssignmentManager.class); Map<String, Map<String, String>> localityMappings = new HashMap<>(); localityMappings.put("0", ImmutableMap.of(SetContainerHostMapping.HOST_KEY, "abc-affinity")); Map<String, String> taskAssignment = ImmutableMap.of("task-0", "0"); // Mock the container locality assignment. when(mockLocalityManager.readContainerLocality()).thenReturn(localityMappings); // Mock the container to task assignment. when(mockTaskAssignmentManager.readTaskAssignment()).thenReturn(taskAssignment); when(mockTaskAssignmentManager.readTaskModes()).thenReturn(Collections.singletonMap(new TaskName("task-0"), TaskMode.Active)); GrouperMetadataImpl grouperMetadata = JobModelManager.getGrouperMetadata(new MapConfig(), mockLocalityManager, mockTaskAssignmentManager); Mockito.verify(mockLocalityManager).readContainerLocality(); Mockito.verify(mockTaskAssignmentManager).readTaskAssignment(); Assert.assertEquals(ImmutableMap.of("0", new LocationId("abc-affinity"), "1", new LocationId("ANY_HOST")), grouperMetadata.getProcessorLocality()); Assert.assertEquals(ImmutableMap.of(new TaskName("task-0"), new LocationId("abc-affinity")), grouperMetadata.getTaskLocality()); }
@Test public void testWriteOnlyLocalityManager() { LocalityManager localityManager = new LocalityManager(config, new MetricsRegistryMap()); localityManager.writeContainerToHostMapping("1", "localhost"); assertEquals(localityManager.readContainerLocality().size(), 1); assertEquals(ImmutableMap.of("1", ImmutableMap.of("host", "localhost")), localityManager.readContainerLocality()); localityManager.close(); MockCoordinatorStreamSystemProducer producer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemProducer(config, null); MockCoordinatorStreamSystemConsumer consumer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemConsumer(config, null); assertTrue(producer.isStopped()); assertTrue(consumer.isStopped()); } }