protected MessageId getMessageId(Collection<Tuple> anchors) { MessageId ret = null; if (anchors != null && ackerNum > 0) { Map<Long, Long> anchors_to_ids = new HashMap<>(); for (Tuple a : anchors) { if (a.getMessageId() != null) { Long edge_id = MessageId.generateId(random); put_xor(pendingAcks, a, edge_id); MessageId messageId = a.getMessageId(); if (messageId != null) { for (Long root_id : messageId.getAnchorsToIds().keySet()) { put_xor(anchors_to_ids, root_id, edge_id); } } } } ret = MessageId.makeId(anchors_to_ids); } return ret; }
public TupleImpl(GeneralTopologyContext context, List<Object> values, int taskId, String streamId) { this(context, values, taskId, streamId, MessageId.makeUnanchored()); }
@Override public String toString() { return "source: " + getSourceComponent() + ":" + taskId + ", stream: " + streamId + ", id: " + id.toString() + ", " + values.toString(); }
protected MessageId getMessageId(SpoutMsgInfo msg, Map<Long, MsgInfo> ackBatch) { MessageId msgId = null; if (msg.rootId != null) { Long as = MessageId.generateId(random); msgId = MessageId.makeRootId(msg.rootId, as); MsgInfo msgInfo = ackBatch.get(msg.rootId); List<Object> ackerTuple; if (msgInfo == null) { TupleInfo info = TupleInfo.buildTupleInfo(msg.streamId, msg.messageId, msg.values, System.currentTimeMillis(), isCacheTuple); pending.putHead(msg.rootId, info); ackerTuple = JStormUtils.mk_list((Object) msg.rootId, JStormUtils.bit_xor_vals(as), task_id); msgInfo = new SpoutMsgInfo(Acker.ACKER_INIT_STREAM_ID, ackerTuple, null, null, null, null); ackBatch.put(msg.rootId, msgInfo); } else { ackerTuple = msgInfo.values; ackerTuple.set(1, JStormUtils.bit_xor_vals(ackerTuple.get(1), as)); } } return msgId; }
long root_id = MessageId.generateId(); Integer askers=StormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_ACKERS)); Boolean isroot = (message_id != null)&&(askers > 0); MessageId msgid; if (isroot) { msgid = MessageId.makeRootId(root_id, t); } else { msgid = MessageId.makeUnanchored();
public static MessageId makeId(Map<Long, Long> anchorsToIds) { return new MessageId(anchorsToIds); }
public static MessageId makeUnanchored() { return makeId(new HashMap<Long, Long>()); }
private boolean sendAckTuple(Tuple input) { boolean ret = false; Integer pendingCount; synchronized (pendingTuples) { pendingCount = pendingTuples.get(input); } if (pendingCount == null || pendingCount <= 0) { long ack_val = 0L; Object pend_val = pendingAcks.remove(input); if (pend_val != null) { ack_val = (Long) (pend_val); } MessageId messageId = input.getMessageId(); if (messageId != null) { for (Map.Entry<Long, Long> e : messageId.getAnchorsToIds().entrySet()) { List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey(), JStormUtils.bit_xor(e.getValue(), ack_val)); sendBoltMsg(Acker.ACKER_ACK_STREAM_ID, null, ackTuple, null, null); } } ret = true; } return ret; }
public byte[] serialize(Tuple tuple) { try { _outputter.reset(); WritableUtils.writeVInt(_dataOutputter, tuple.getSourceTask()); WritableUtils.writeVInt(_dataOutputter, _ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId())); tuple.getMessageId().serialize(_dataOutputter); _kryo.serializeInto(tuple.getValues(), _outputter); return _outputter.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); } }
public Tuple deserialize(byte[] ser) { try { ByteArrayInputStream bin = new ByteArrayInputStream(ser); DataInputStream in = new DataInputStream(bin); int taskId = WritableUtils.readVInt(in); int streamId = WritableUtils.readVInt(in); String componentName = _context.getComponentId(taskId); String streamName = _ids.getStreamName(componentName, streamId); MessageId id = MessageId.deserialize(in); List<Object> values = _kryo.deserializeFrom(bin); return new Tuple(_context, values, taskId, streamName, id); } catch(IOException e) { throw new RuntimeException(e); } } }
protected Long getRootId(Object messageId) { Boolean needAck = (messageId != null) && (ackerNum > 0); // This change storm logic // Storm can't make sure root_id is unique // storm's logic is root_id = MessageId.generateId(random); // when duplicate root_id, it will miss call ack/fail Long rootId = null; if (needAck) { rootId = MessageId.generateId(random); /* while (pending.containsKey(rootId)) { rootId = MessageId.generateId(random); }*/ } return rootId; }
public static boolean isDebugRecv(MessageId msgId) { return msgId != null && isDebugRecv(msgId.getAnchors()); }
if (needAck) { Long as = MessageId.generateId(random); msgId = MessageId.makeRootId(root_id, as); ackSeq.add(as); } else {
public static MessageId makeId(Map<Long, Long> anchorsToIds) { return new MessageId(anchorsToIds); }
public static MessageId makeUnanchored() { return makeId(new HashMap<Long, Long>()); }
@Override public void fail(Tuple input) { // if ackerNum == 0, we can just return if (input.getMessageId() != null) { pendingAcks.remove(input); for (Map.Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey()); sendBoltMsg(Acker.ACKER_FAIL_STREAM_ID, null, ackTuple, null, null); } } taskStats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId()); }
Pair<MessageId, List<Object>> pairValue = (Pair<MessageId, List<Object>>) value; if (pairValue.getFirst() != null) { pairValue.getFirst().serialize(output); } else { output.writeInt(0, true); MessageId msgId = tuple.getMessageId(); if (msgId != null) { msgId.serialize(output); } else { output.writeInt(0, true);
if (_ackerNum > 0) { for (int i = 0; i < len; i++) { values.add(new Pair<>(MessageId.deserialize(input), _kryo.deserializeFrom(input))); id = MessageId.deserialize(input); values = _kryo.deserializeFrom(input);
public void execute(Tuple input) { long genId = MessageId.generateId(); _inputs.put(genId, input); try {
/** * one of the tuples has been chosen, the logs should be output */ private static boolean sample(Collection<Tuple> anchors) { if (Double.compare(sampleRate, 1.0d) >= 0) return true; for (Tuple t : anchors) { if (sample(t.getMessageId().getAnchors())) { return true; } } return false; }