/** * Get the class name according to the serde name. * * @param name serde name * @return serde factory name, or null if there is no factory defined for the * supplied serde name. */ public String getSerdeClass(String name) { return get(String.format(SerializerConfig.SERDE_FACTORY_CLASS(), name), null); }
private Map<String, Serde<Object>> getSerdes() { Map<String, Serde<Object>> serdeMap = new HashMap<>(); SerializerConfig serializerConfig = new SerializerConfig(jobConfig); // Adding all serdes from factories ScalaJavaUtil.toJavaCollection(serializerConfig.getSerdeNames()) .stream() .forEach(serdeName -> { Option<String> serdeClassName = serializerConfig.getSerdeClass(serdeName); if (serdeClassName.isEmpty()) { serdeClassName = Option.apply(SerializerConfig.getSerdeFactoryName(serdeName)); } Serde serde = Util.getObj(serdeClassName.get(), SerdeFactory.class).getSerde(serdeName, serializerConfig); serdeMap.put(serdeName, serde); }); return serdeMap; }
String serdeName = serdeUUIDs.computeIfAbsent(serde, s -> serde.getClass().getSimpleName() + "-" + UUID.randomUUID().toString()); serdeConfigs.putIfAbsent(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeName), base64Encoder.encodeToString(serializableSerde.toBytes(serde))); });
private Map<String, Serde> validateAndGetDeserializedSerdes(Config jobConfig, int numSerdes) { Config serializers = jobConfig.subset("serializers.registry.", true); // make sure that the serializers deserialize correctly SerializableSerde<Serde> serializableSerde = new SerializableSerde<>(); assertEquals(numSerdes, serializers.size()); return serializers.entrySet().stream().collect(Collectors.toMap( e -> e.getKey().replace(SerializerConfig.SERIALIZED_INSTANCE_SUFFIX(), ""), e -> serializableSerde.fromBytes(Base64.getDecoder().decode(e.getValue().getBytes())) )); }
private void addValueSerde(Map<String, String> map) { String serdeId = "value-serde"; map.put(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeId), serializeSerde(new StringSerde("UTF-8"))); map.put(String.format(JavaTableConfig.STORE_MSG_SERDE, TABLE_ID), serdeId); }
private void addKeySerde(Map<String, String> map) { String serdeId = "key-serde"; map.put(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeId), serializeSerde(new IntegerSerde())); map.put(String.format(JavaTableConfig.STORE_KEY_SERDE, TABLE_ID), serdeId); }
/** * Get the class name according to the serde name. * * @param name serde name * @return serde factory name, or null if there is no factory defined for the * supplied serde name. */ public String getSerdeClass(String name) { return get(String.format(SerializerConfig.SERDE_FACTORY_CLASS(), name), null); }
String serdeName = serdeUUIDs.computeIfAbsent(serde, s -> serde.getClass().getSimpleName() + "-" + UUID.randomUUID().toString()); configs.putIfAbsent(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeName), base64Encoder.encodeToString(serializableSerde.toBytes(serde))); });
/** * set the serde for this appender. It looks for the stream serde first, then system serde. * If still can not get the serde, throws exceptions. * * @param log4jSystemConfig log4jSystemConfig for this appender * @param systemName name of the system * @param streamName name of the stream */ private void setSerde(Log4jSystemConfig log4jSystemConfig, String systemName, String streamName) { String serdeClass = LoggingEventJsonSerdeFactory.class.getCanonicalName(); String serdeName = log4jSystemConfig.getStreamSerdeName(systemName, streamName); if (serdeName != null) { serdeClass = log4jSystemConfig.getSerdeClass(serdeName); } if (serdeClass != null) { SerdeFactory<LogEvent> serdeFactory = Util.getObj(serdeClass, SerdeFactory.class); serde = serdeFactory.getSerde(systemName, config); } else { String serdeKey = String.format(SerializerConfig.SERDE_FACTORY_CLASS(), serdeName); throw new SamzaException("Can not find serializers class for key '" + serdeName + "'. Please specify " + serdeKey + " property"); } }
String serdeName = serdeUUIDs.computeIfAbsent(serde, s -> serde.getClass().getSimpleName() + "-" + UUID.randomUUID().toString()); configs.putIfAbsent(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeName), base64Encoder.encodeToString(serializableSerde.toBytes(serde))); });
/** * set the serde for this appender. It looks for the stream serde first, then system serde. * If still can not get the serde, throws exceptions. * * @param log4jSystemConfig log4jSystemConfig for this appender * @param systemName name of the system * @param streamName name of the stream */ private void setSerde(Log4jSystemConfig log4jSystemConfig, String systemName, String streamName) { String serdeClass = LoggingEventJsonSerdeFactory.class.getCanonicalName(); String serdeName = log4jSystemConfig.getStreamSerdeName(systemName, streamName); if (serdeName != null) { serdeClass = log4jSystemConfig.getSerdeClass(serdeName); } if (serdeClass != null) { SerdeFactory<LoggingEvent> serdeFactory = Util.getObj(serdeClass, SerdeFactory.class); serde = serdeFactory.getSerde(systemName, config); } else { String serdeKey = String.format(SerializerConfig.SERDE_FACTORY_CLASS(), serdeName); throw new SamzaException("Can not find serializers class for key '" + serdeName + "'. Please specify " + serdeKey + " property"); } }
String serdeName = serdeUUIDs.computeIfAbsent(serde, s -> serde.getClass().getSimpleName() + "-" + UUID.randomUUID().toString()); configs.putIfAbsent(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeName), base64Encoder.encodeToString(serializableSerde.toBytes(serde))); });
String serdeName = serdeUUIDs.computeIfAbsent(serde, s -> serde.getClass().getSimpleName() + "-" + UUID.randomUUID().toString()); configs.putIfAbsent(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeName), base64Encoder.encodeToString(serializableSerde.toBytes(serde))); });
String serdeName = serdeUUIDs.computeIfAbsent(serde, s -> serde.getClass().getSimpleName() + "-" + UUID.randomUUID().toString()); configs.putIfAbsent(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeName), base64Encoder.encodeToString(serializableSerde.toBytes(serde))); });