private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
@Test public void testWriteOnlyLocalityManager() { LocalityManager localityManager = new LocalityManager(config, new MetricsRegistryMap()); localityManager.writeContainerToHostMapping("1", "localhost"); assertEquals(localityManager.readContainerLocality().size(), 1); assertEquals(ImmutableMap.of("1", ImmutableMap.of("host", "localhost")), localityManager.readContainerLocality()); localityManager.close(); MockCoordinatorStreamSystemProducer producer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemProducer(config, null); MockCoordinatorStreamSystemConsumer consumer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemConsumer(config, null); assertTrue(producer.isStopped()); assertTrue(consumer.isStopped()); } }
/** * Builds list of {@link Task} from job model in coordinator stream. * @param consumer system consumer associated with a job's coordinator stream. * @return list of {@link Task} constructed from job model in coordinator stream. */ protected List<Task> readTasksFromCoordinatorStream(CoordinatorStreamSystemConsumer consumer) { LocalityManager localityManager = new LocalityManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, Map<String, String>> containerIdToHostMapping = localityManager.readContainerLocality(); TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, String> taskNameToContainerIdMapping = taskAssignmentManager.readTaskAssignment(); StorageConfig storageConfig = new StorageConfig(consumer.getConfig()); List<String> storeNames = JavaConverters.seqAsJavaListConverter(storageConfig.getStoreNames()).asJava(); return taskNameToContainerIdMapping.entrySet() .stream() .map(entry -> { String hostName = containerIdToHostMapping.get(entry.getValue()).get(SetContainerHostMapping.HOST_KEY); return new Task(hostName, entry.getKey(), entry.getValue(), new ArrayList<>(), storeNames); }).collect(Collectors.toList()); } }
Optional<ExternalContext> externalContextOptional) { TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(appDesc); LocalityManager localityManager = new LocalityManager(config, new MetricsRegistryMap()); SamzaContainer container = SamzaContainer$.MODULE$.apply( containerId,
@Test public void testLocalityManager() { LocalityManager localityManager = new LocalityManager(config, new MetricsRegistryMap()); localityManager.writeContainerToHostMapping("0", "localhost"); Map<String, Map<String, String>> localMap = localityManager.readContainerLocality(); Map<String, Map<String, String>> expectedMap = new HashMap<String, Map<String, String>>() { { this.put("0", new HashMap<String, String>() { { this.put(SetContainerHostMapping.HOST_KEY, "localhost"); } }); } }; assertEquals(expectedMap, localMap); localityManager.close(); MockCoordinatorStreamSystemProducer producer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemProducer(config, null); MockCoordinatorStreamSystemConsumer consumer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemConsumer(config, null); assertTrue(producer.isStopped()); assertTrue(consumer.isStopped()); }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
private void populateContainerLocalityMappings() { Map<String, Map<String, String>> allMappings = localityManager.readContainerLocality(); for (String containerId: containers.keySet()) { if (allMappings.containsKey(containerId)) { localityMappings.put(containerId, allMappings.get(containerId).get(SetContainerHostMapping.HOST_KEY)); } else { localityMappings.put(containerId, null); } } }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
public Map<String, String> getAllContainerToHostValues(String key) { if (localityManager == null) { return Collections.EMPTY_MAP; } Map<String, String> allValues = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : localityManager.readContainerLocality().entrySet()) { String value = entry.getValue().get(key); if (value != null) { allValues.put(entry.getKey(), value); } } return allValues; }
/** * Returns the container to host mapping for a given container ID and mapping key * * @param containerId the ID of the container * @param key mapping key which is one of the keys declared in {@link org.apache.samza.coordinator.stream.messages.SetContainerHostMapping} * @return the value if it exists for a given container and key, otherwise an empty string */ public String getContainerToHostValue(String containerId, String key) { if (localityManager == null) { return EMPTY_STRING; } final Map<String, String> mappings = localityManager.readContainerLocality().get(containerId); if (mappings == null) { return EMPTY_STRING; } if (!mappings.containsKey(key)) { return EMPTY_STRING; } return mappings.get(key); }
/** * Returns the container to host mapping for a given container ID and mapping key * * @param containerId the ID of the container * @param key mapping key which is one of the keys declared in {@link org.apache.samza.coordinator.stream.messages.SetContainerHostMapping} * @return the value if it exists for a given container and key, otherwise an empty string */ public String getContainerToHostValue(String containerId, String key) { if (localityManager == null) { return EMPTY_STRING; } final Map<String, String> mappings = localityManager.readContainerLocality().get(containerId); if (mappings == null) { return EMPTY_STRING; } if (!mappings.containsKey(key)) { return EMPTY_STRING; } return mappings.get(key); }
/** * Returns the container to host mapping for a given container ID and mapping key * * @param containerId the ID of the container * @param key mapping key which is one of the keys declared in {@link org.apache.samza.coordinator.stream.messages.SetContainerHostMapping} * @return the value if it exists for a given container and key, otherwise an empty string */ public String getContainerToHostValue(String containerId, String key) { if (localityManager == null) { return EMPTY_STRING; } final Map<String, String> mappings = localityManager.readContainerLocality().get(containerId); if (mappings == null) { return EMPTY_STRING; } if (!mappings.containsKey(key)) { return EMPTY_STRING; } return mappings.get(key); }
/** * Returns the container to host mapping for a given container ID and mapping key * * @param containerId the ID of the container * @param key mapping key which is one of the keys declared in {@link org.apache.samza.coordinator.stream.messages.SetContainerHostMapping} * @return the value if it exists for a given container and key, otherwise an empty string */ public String getContainerToHostValue(String containerId, String key) { if (localityManager == null) { return EMPTY_STRING; } final Map<String, String> mappings = localityManager.readContainerLocality().get(containerId); if (mappings == null) { return EMPTY_STRING; } if (!mappings.containsKey(key)) { return EMPTY_STRING; } return mappings.get(key); }
/** * Returns the container to host mapping for a given container ID and mapping key * * @param containerId the ID of the container * @param key mapping key which is one of the keys declared in {@link org.apache.samza.coordinator.stream.messages.SetContainerHostMapping} * @return the value if it exists for a given container and key, otherwise an empty string */ public String getContainerToHostValue(String containerId, String key) { if (localityManager == null) { return EMPTY_STRING; } final Map<String, String> mappings = localityManager.readContainerLocality().get(containerId); if (mappings == null) { return EMPTY_STRING; } if (!mappings.containsKey(key)) { return EMPTY_STRING; } return mappings.get(key); }
/** * Method to write locality information to the {@link MetadataStore}. This method is used in {@link SamzaContainer}. * * @param containerId the {@link SamzaContainer} ID * @param hostName the hostname */ public void writeContainerToHostMapping(String containerId, String hostName) { Map<String, Map<String, String>> containerToHostMapping = readContainerLocality(); Map<String, String> existingMappings = containerToHostMapping.get(containerId); String existingHostMapping = existingMappings != null ? existingMappings.get(SetContainerHostMapping.HOST_KEY) : null; if (existingHostMapping != null && !existingHostMapping.equals(hostName)) { LOG.info("Container {} moved from {} to {}", new Object[]{containerId, existingHostMapping, hostName}); } else { LOG.info("Container {} started at {}", containerId, hostName); } metadataStore.put(containerId, valueSerde.toBytes(hostName)); }