/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * Return the configuration value as a string. * @return the configuration as a string */ public String getConfigValue() { return getMessageValue(CONFIG_VALUE_KEY); } }
/** * The SetConfig message is used to store a specific configuration. * This constructor is used to create a SetConfig message for a given source for a specific config key and config value. * @param source the source of the config * @param key the key for the given config * @param value the value for the given config */ public SetConfig(String source, String key, String value) { super(source); setType(TYPE); setKey(key); putMessageValue(CONFIG_VALUE_KEY, value); }
@Override public String fromBytes(byte[] bytes) { Map<String, Object> values = messageSerde.fromBytes(bytes); CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[]{}, values); if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(message); return hostMapping.getHostLocality(); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message); return setTaskContainerMapping.getTaskAssignment(); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(message); return String.valueOf(changelogMapping.getPartition()); } else if (type.equalsIgnoreCase(SetConfig.TYPE)) { SetConfig setConfig = new SetConfig(message); return setConfig.getConfigValue(); } else if (type.equalsIgnoreCase(SetTaskModeMapping.TYPE)) { SetTaskModeMapping setTaskModeMapping = new SetTaskModeMapping(message); return String.valueOf(setTaskModeMapping.getTaskMode()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } }
@Test public void testSetConfig() { SetConfig setConfig = new SetConfig("source", "key", "value"); assertEquals(SetConfig.TYPE, setConfig.getType()); assertEquals("key", setConfig.getKey()); assertEquals("value", setConfig.getConfigValue()); assertFalse(setConfig.isDelete()); assertEquals(CoordinatorStreamMessage.VERSION, setConfig.getVersion()); }
String key = null; try { SetConfig setConfigMessage = new SetConfig(message); key = setConfigMessage.getKey(); Map<String, String> valuesMap = (Map<String, String>) setConfigMessage.getMessageMap().get("values"); String value = null; if (valuesMap != null) {
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> map = new LinkedHashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>(); assertEquals(1, systemStreamPartitions.size()); SystemStreamPartition systemStreamPartition = systemStreamPartitions.iterator().next(); assertEquals(expectedSystemStreamPartition, systemStreamPartition); if (pollCount++ == 0) { List<IncomingMessageEnvelope> list = new ArrayList<IncomingMessageEnvelope>(); SetConfig setConfig1 = new SetConfig("test", "job.name", "my-job-name"); SetConfig setConfig2 = new SetConfig("test", "job.id", "1234"); Delete delete = new Delete("test", "job.name", SetConfig.TYPE); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig1.getKeyArray()), serialize(setConfig1.getMessageMap()))); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig2.getKeyArray()), serialize(setConfig2.getMessageMap()))); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(delete.getKeyArray()), delete.getMessageMap())); map.put(systemStreamPartition, list); } return map; }
@Test public void testHashCodeAndEquality() { SetConfig message = new SetConfig("source", "key1", "value1"); SetConfig message1 = new SetConfig("source", "key1", "value1"); SetConfig message2 = new SetConfig("source", "key2", "value1"); assertEquals(message.hashCode(), message1.hashCode()); assertEquals(message, message1); assertTrue(!message.equals(message2)); } }
@Override public byte[] toBytes(String value) { if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) { SetContainerHostMapping hostMapping = new SetContainerHostMapping(SOURCE, "", value, "", ""); return messageSerde.toBytes(hostMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) { SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(SOURCE, "", value); return messageSerde.toBytes(setTaskContainerMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetTaskModeMapping.TYPE)) { SetTaskModeMapping setTaskModeMapping = new SetTaskModeMapping(SOURCE, "", value); return messageSerde.toBytes(setTaskModeMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) { SetChangelogMapping changelogMapping = new SetChangelogMapping(SOURCE, "", Integer.valueOf(value)); return messageSerde.toBytes(changelogMapping.getMessageMap()); } else if (type.equalsIgnoreCase(SetConfig.TYPE)) { SetConfig setConfig = new SetConfig(SOURCE, "", value); return messageSerde.toBytes(setConfig.getMessageMap()); } else { throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type)); } } }
configMap.remove(configKey); } else { String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue(); configMap.put(configKey, configValue);
private void convertConfigToCoordinatorMessage(Config config) { try { for (Map.Entry<String, String> configPair : config.entrySet()) { byte[] keyBytes = null; byte[] messgeBytes = null; if (configPair.getKey().startsWith(CHANGELOGPREFIX)) { String[] changelogInfo = configPair.getKey().split(":"); String changeLogPartition = configPair.getValue(); SetChangelogMapping changelogMapping = new SetChangelogMapping(changelogInfo[1], changelogInfo[2], Integer.parseInt(changeLogPartition)); keyBytes = MAPPER.writeValueAsString(changelogMapping.getKeyArray()).getBytes("UTF-8"); messgeBytes = MAPPER.writeValueAsString(changelogMapping.getMessageMap()).getBytes("UTF-8"); } else { SetConfig setConfig = new SetConfig("source", configPair.getKey(), configPair.getValue()); keyBytes = MAPPER.writeValueAsString(setConfig.getKeyArray()).getBytes("UTF-8"); messgeBytes = MAPPER.writeValueAsString(setConfig.getMessageMap()).getBytes("UTF-8"); } // The ssp here is the coordinator ssp (which is always fixed) and not the task ssp. put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, "", keyBytes, messgeBytes)); } setIsAtHead(systemStreamPartition, true); } catch (Exception e) { throw new SamzaException(e); } }
/** * The SetConfig message is used to store a specific configuration. * This constructor is used to create a SetConfig message for a given source for a specific config key and config value. * @param source the source of the config * @param key the key for the given config * @param value the value for the given config */ public SetConfig(String source, String key, String value) { super(source); setType(TYPE); setKey(key); putMessageValue(CONFIG_VALUE_KEY, value); }
configMap.remove(configKey); } else { String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue(); configMap.put(configKey, configValue);
/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * Return the configuration value as a string. * @return the configuration as a string */ public String getConfigValue() { return getMessageValue(CONFIG_VALUE_KEY); } }
/** * The SetConfig message is used to store a specific configuration. * This constructor is used to create a SetConfig message for a given source for a specific config key and config value. * @param source the source of the config * @param key the key for the given config * @param value the value for the given config */ public SetConfig(String source, String key, String value) { super(source); setType(TYPE); setKey(key); putMessageValue(CONFIG_VALUE_KEY, value); }
configMap.remove(configKey); } else { String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue(); configMap.put(configKey, configValue);
/** * This method sends message of type "set-config" to the coordinator stream * * @param key defines the name of the configuration being set. For example, for setting the number of yarn containers, * the key is "yarn.container.count" * @param value defines the value associated with the key. For example, if the key is "yarn.container.count" the value * is the new number of containers. */ private void sendSetConfigMessage(String key, String value) { log.info("sent SetConfig message with key = " + key + " and value = " + value); coordinatorStreamSystemProducer.send(new SetConfig(CoordinatorStreamWriter.SOURCE, key, value)); }
/** * Return the configuration value as a string. * @return the configuration as a string */ public String getConfigValue() { return getMessageValue(CONFIG_VALUE_KEY); } }
/** * The SetConfig message is used to store a specific configuration. * This constructor is used to create a SetConfig message for a given source for a specific config key and config value. * @param source the source of the config * @param key the key for the given config * @param value the value for the given config */ public SetConfig(String source, String key, String value) { super(source); setType(TYPE); setKey(key); putMessageValue(CONFIG_VALUE_KEY, value); }