@Override protected KryoValuesDeserializer initialValue() { return new KryoValuesDeserializer(conf); } };
private List<Object> deserialize(byte[] bytes, Map<String, Object> conf) throws IOException { KryoValuesDeserializer deserializer = new KryoValuesDeserializer(mkConf(conf)); return deserializer.deserialize(bytes); }
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { Input in = new Input(request.getInputStream()); List<Object> metrics = des.get().deserializeFrom(in); handle((TaskInfo)metrics.get(0), (Collection<DataPoint>)metrics.get(1), (String)metrics.get(2)); response.setStatus(HttpServletResponse.SC_OK); } }
@Test public void testSerialization() throws IOException { Map conf = Utils.readDefaultConfig(); Config.registerSerialization(conf, Metadata.class); KryoValuesSerializer kvs = new KryoValuesSerializer(conf); Metadata md = new Metadata(); byte[] content = kvs.serializeObject(md); KryoValuesDeserializer kvd = new KryoValuesDeserializer(conf); Metadata md2 = (Metadata) kvd.deserializeObject(content); // TODO compare md1 and md2 } }
public static BackPressureStatus read(byte[] bytes, KryoValuesDeserializer deserializer) { return (BackPressureStatus) deserializer.deserializeObject(bytes); }
throw new RuntimeException("Metrics messages are sent to the system task (" + client.getDstAddress() + ") " + tm); List<Object> metrics = _des.deserialize(tm.message()); if (metrics.size() < 1) { throw new RuntimeException("No metrics data in the metrics message (" + client.getDstAddress() + ") " + metrics);
public Object getData(String path) { path = "/" + path; try { if(_curator.checkExists().forPath(path)!=null) { return _des.deserializeObject(_curator.getData().forPath(path)); } else { return null; } } catch(Exception e) { throw new RuntimeException(e); } }
@Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent event) { //examine the response message from server Object message = event.getMessage(); if (message instanceof ControlMessage) { ControlMessage msg = (ControlMessage)message; if (msg==ControlMessage.FAILURE_RESPONSE) { LOG.info("failure response:{}", msg); } } else if (message instanceof List) { try { //This should be the metrics, and there should only be one of them List<TaskMessage> list = (List<TaskMessage>)message; if (list.size() < 1) throw new RuntimeException("Didn't see enough load metrics ("+client.getDstAddress()+") "+list); if (list.size() != 1) LOG.debug("Messages are not being delivered fast enough, got "+list.size()+" metrics messages at once("+client.getDstAddress()+")"); TaskMessage tm = ((List<TaskMessage>)message).get(list.size() - 1); if (tm.task() != -1) throw new RuntimeException("Metrics messages are sent to the system task ("+client.getDstAddress()+") "+tm); List metrics = _des.deserialize(tm.message()); if (metrics.size() < 1) throw new RuntimeException("No metrics data in the metrics message ("+client.getDstAddress()+") "+metrics); if (!(metrics.get(0) instanceof Map)) throw new RuntimeException("The metrics did not have a map in the first slot ("+client.getDstAddress()+") "+metrics); client.setLoadMetrics((Map<Integer, Double>)metrics.get(0)); } catch (IOException e) { throw new RuntimeException(e); } } else { throw new RuntimeException("Don't know how to handle a message of type " + message + " (" + client.getDstAddress() + ")"); } }
StormClientHandler(Client client, AtomicBoolean[] remoteBpStatus, Map<String, Object> conf) { this.client = client; this.remoteBpStatus = remoteBpStatus; _des = new KryoValuesDeserializer(conf); }
public List<Object> deserialize(byte[] ser) { _kryoInput.setBuffer(ser); return deserializeFrom(_kryoInput); }
public KryoTupleDeserializer(final Map<String, Object> conf, final GeneralTopologyContext context) { _kryo = new KryoValuesDeserializer(conf); _context = context; _ids = new SerializationFactory.IdDictionary(context.getRawTopology()); _kryoInput = new Input(1); }
@Override public TupleImpl deserialize(byte[] ser) { try { _kryoInput.setBuffer(ser); int taskId = _kryoInput.readInt(true); int streamId = _kryoInput.readInt(true); String componentName = _context.getComponentId(taskId); String streamName = _ids.getStreamName(componentName, streamId); MessageId id = MessageId.deserialize(_kryoInput); List<Object> values = _kryo.deserializeFrom(_kryoInput); return new TupleImpl(_context, values, componentName, taskId, streamName, id); } catch (IOException e) { throw new RuntimeException(e); } } }
@Override protected void initChannel(Channel ch) throws Exception { // Create a default pipeline implementation. ChannelPipeline pipeline = ch.pipeline(); // Decoder pipeline.addLast("decoder", new MessageDecoder(new KryoValuesDeserializer(conf))); // Encoder pipeline.addLast("encoder", NettySerializableMessageEncoder.INSTANCE); boolean isNettyAuth = (Boolean) conf .get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION); if (isNettyAuth) { // Authenticate: Removed after authentication completes pipeline.addLast("saslClientHandler", new SaslStormClientHandler( client)); } // business logic. pipeline.addLast("handler", new StormClientHandler(client, remoteBpStatus, conf)); } }
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { Input in = new Input(request.getInputStream()); List<Object> metrics = _des.get().deserializeFrom(in); handle((TaskInfo)metrics.get(0), (Collection<DataPoint>)metrics.get(1)); response.setStatus(HttpServletResponse.SC_OK); } }
@Override protected void initChannel(Channel ch) throws Exception { // Create a default pipeline implementation. ChannelPipeline pipeline = ch.pipeline(); // Decoder pipeline.addLast("decoder", new MessageDecoder(new KryoValuesDeserializer(topoConf))); // Encoders pipeline.addLast("netty-serializable-encoder", NettySerializableMessageEncoder.INSTANCE); pipeline.addLast("backpressure-encoder", new BackPressureStatusEncoder(new KryoValuesSerializer(topoConf))); boolean isNettyAuth = (Boolean) topoConf .get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION); if (isNettyAuth) { // Authenticate: Removed after authentication completes pipeline.addLast("saslServerHandler", new SaslStormServerHandler( server)); // Authorize pipeline.addLast("authorizeServerHandler", new SaslStormServerAuthorizeHandler()); } // business logic. pipeline.addLast("handler", new StormServerHandler(server)); } }
public List<Object> deserialize(byte[] ser) throws IOException { _kryoInput.setBuffer(ser); return deserializeFrom(_kryoInput); }
@Override protected KryoValuesDeserializer initialValue() { return new KryoValuesDeserializer(_conf); } };
public Tuple deserialize(byte[] ser) { try { _kryoInput.setBuffer(ser); int taskId = _kryoInput.readInt(true); int streamId = _kryoInput.readInt(true); String componentName = _context.getComponentId(taskId); String streamName = _ids.getStreamName(componentName, streamId); MessageId id = MessageId.deserialize(_kryoInput); List<Object> values = _kryo.deserializeFrom(_kryoInput); return new TupleImpl(_context, values, taskId, streamName, id); } catch(IOException e) { throw new RuntimeException(e); } } }
StormClientHandler(Client client, Map conf) { this.client = client; _des = new KryoValuesDeserializer(conf); }
public KryoTupleDeserializer(final Map conf, final GeneralTopologyContext context) { _kryo = new KryoValuesDeserializer(conf); _context = context; _ids = new SerializationFactory.IdDictionary(context.getRawTopology()); _kryoInput = new Input(1); }