/** * one of the tuples has been chosen, the logs should be output */ private static boolean sample(Collection<Tuple> anchors) { if (Double.compare(sampleRate, 1.0d) >= 0) return true; for (Tuple t : anchors) { if (sample(t.getMessageId().getAnchors())) { return true; } } return false; }
@Override public void fail(Tuple input) { // if ackerNum == 0, we can just return if (input.getMessageId() != null) { pendingAcks.remove(input); for (Map.Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey()); sendBoltMsg(Acker.ACKER_FAIL_STREAM_ID, null, ackTuple, null, null); } } taskStats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId()); }
protected MessageId getMessageId(Collection<Tuple> anchors) { MessageId ret = null; if (anchors != null && ackerNum > 0) { Map<Long, Long> anchors_to_ids = new HashMap<>(); for (Tuple a : anchors) { if (a.getMessageId() != null) { Long edge_id = MessageId.generateId(random); put_xor(pendingAcks, a, edge_id); MessageId messageId = a.getMessageId(); if (messageId != null) { for (Long root_id : messageId.getAnchorsToIds().keySet()) { put_xor(anchors_to_ids, root_id, edge_id); } } } } ret = MessageId.makeId(anchors_to_ids); } return ret; }
@Override public void fail(Tuple input) { // if ackerNum == 0, we can just return if (input.getMessageId() != null) { pendingAcks.remove(input); for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { unanchoredSend(topologyContext, sendTargets, taskTransfer, Acker.ACKER_FAIL_STREAM_ID, JStormUtils.mk_list((Object) e.getKey())); } } tupleStartTimes.remove(input); taskStats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId()); }
private boolean sendAckTuple(Tuple input) { boolean ret = false; Integer pendingCount; synchronized (pendingTuples) { pendingCount = pendingTuples.get(input); } if (pendingCount == null || pendingCount <= 0) { long ack_val = 0L; Object pend_val = pendingAcks.remove(input); if (pend_val != null) { ack_val = (Long) (pend_val); } MessageId messageId = input.getMessageId(); if (messageId != null) { for (Map.Entry<Long, Long> e : messageId.getAnchorsToIds().entrySet()) { List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey(), JStormUtils.bit_xor(e.getValue(), ack_val)); sendBoltMsg(Acker.ACKER_ACK_STREAM_ID, null, ackTuple, null, null); } } ret = true; } return ret; }
@Override public void ack(Tuple input) { if (input.getMessageId() != null) { Long ack_val = 0L; Object pend_val = pendingAcks.remove(input); if (pend_val != null) { ack_val = (Long) (pend_val); } for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { unanchoredSend(topologyContext, sendTargets, taskTransfer, Acker.ACKER_ACK_STREAM_ID, JStormUtils.mk_list((Object) e.getKey(), JStormUtils.bit_xor(e.getValue(), ack_val))); } } Long latencyStart = (Long) tupleStartTimes.remove(input); taskStats.bolt_acked_tuple(input.getSourceComponent(), input.getSourceStreamId()); if (latencyStart != null && JStormMetrics.enabled) { long endTime = System.currentTimeMillis(); taskStats.update_bolt_acked_latency(input.getSourceComponent(), input.getSourceStreamId(), latencyStart, endTime); } }
public byte[] serialize(Tuple tuple) { try { _outputter.reset(); WritableUtils.writeVInt(_dataOutputter, tuple.getSourceTask()); WritableUtils.writeVInt(_dataOutputter, _ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId())); tuple.getMessageId().serialize(_dataOutputter); _kryo.serializeInto(tuple.getValues(), _outputter); return _outputter.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); } }
private void processTupleEvent(Tuple tuple) { if (tuple.getMessageId() != null && tuple.getMessageId().isAnchored()) { tupleStartTimes.put(tuple, System.currentTimeMillis()); } try { // for watermarks, just forward to downstream operators if (!isSystemBolt && Common.WATERMARK_STREAM_ID.equals(tuple.getSourceStreamId())) { outputCollector.emit(Common.WATERMARK_STREAM_ID, tuple.getValues()); } if (!isSystemBolt && tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID)) { TopoMasterCtrlEvent event = (TopoMasterCtrlEvent) tuple.getValue(0); if (event.isTransactionEvent()) { bolt.execute(tuple); } else { LOG.warn("Received an unexpected control event, {}", event); } } else if (tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_REGISTER_METRICS_RESP_STREAM_ID)) { this.metricsReporter.updateMetricMeta((Map<String, Long>) tuple.getValue(0)); } else { bolt.execute(tuple); } } catch (Throwable e) { error = e; LOG.error("bolt execute error ", e); reportError.report(e); } }
@Override public void ack(Tuple input) { if (input.getMessageId() != null) { if (!sendAckTuple(input)) { pendingAckQueue.add(input); } } Long latencyStart = (Long) tupleStartTimes.remove(input); taskStats.bolt_acked_tuple(input.getSourceComponent(), input.getSourceStreamId()); if (latencyStart != null && JStormMetrics.enabled) { long endTime = System.currentTimeMillis(); taskStats.update_bolt_acked_latency(input.getSourceComponent(), input.getSourceStreamId(), latencyStart, endTime); } }
@Override protected MessageId getMessageId(Collection<Tuple> anchors) { MessageId ret = null; if (anchors != null) { Map<Long, Long> anchors_to_ids = new HashMap<Long, Long>(); long now = System.currentTimeMillis(); if (now - lastRotate > rotateTime) { pendingAcks.rotate(); synchronized (pendingTuples) { pendingTuples.rotate(); } lastRotate = now; } for (Tuple a : anchors) { // Long edge_id = MessageId.generateId(); Long edge_id = MessageId.generateId(random); synchronized (pendingAcks) { put_xor(pendingAcks, a, edge_id); } MessageId messageId = a.getMessageId(); if (messageId != null) { for (Long root_id : messageId.getAnchorsToIds().keySet()) { put_xor(anchors_to_ids, root_id, edge_id); } } } ret = MessageId.makeId(anchors_to_ids); } return ret; }
@Override public void fail(Tuple input) { pending_acks.remove(input); for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { UnanchoredSend.send(topologyContext, sendTargets, workerTransfer, Acker.ACKER_FAIL_STREAM_ID, StormUtils.mk_list((Object) e.getKey())); } Long delta = TasksCommon.tuple_time_delta(tuple_start_times,input); if (delta != null) { Stats.bolt_failed_tuple((BoltTaskStatsRolling) task_stats, input.getSourceComponent(), input.getSourceStreamId(), delta); } }
public void deserializeTuple(KryoTupleDeserializer deserializer, byte[] serMsg, DisruptorQueue queue) { Tuple tuple = deserializer.deserialize(serMsg); if (tuple != null) { if (JStormDebugger.isDebugRecv(tuple.getMessageId())) { LOG.info(idStr + " receive " + tuple.toString()); } //queue.publish(tuple); if (isBackpressureEnable) { if (backpressureStatus) { while (queue.pctFull() > lowMark) { JStormUtils.sleepMs(1); } queue.publish(tuple); backpressureStatus = false; } else { queue.publish(tuple); if (queue.pctFull() > highMark) { backpressureStatus = true; } } } else { queue.publish(tuple); } } } }
@Override public void ack(Tuple input) { Object ack_val = pending_acks.remove(input); if (ack_val == null) { ack_val = 0l; } for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { UnanchoredSend.send( topologyContext, sendTargets, workerTransfer, Acker.ACKER_ACK_STREAM_ID, StormUtils.mk_list((Object)e.getKey(),StormUtils.bit_xor(e.getValue(), ack_val))); } Long delta = TasksCommon.tuple_time_delta(tuple_start_times, input); if (delta != null) { Stats.bolt_acked_tuple((BoltTaskStatsRolling) task_stats, input.getSourceComponent(), input.getSourceStreamId(), delta); } }
private List<Integer> boltEmit(String out_stream_id,Collection<Tuple> anchors, List<Object> values, Integer out_task_id) { try { java.util.Set<Integer> out_tasks = null; if (out_task_id != null) { out_tasks = sendTargets.get(out_task_id, out_stream_id, values); } else { out_tasks = sendTargets.get(out_stream_id, values); } for (Integer t : out_tasks) { HashMap anchors_to_ids = new HashMap(); if (anchors != null) { for (Tuple a : anchors) { Long edge_id = MessageId.generateId(); TasksCommon.put_xor(pending_acks, a, edge_id); for (Long root_id : a.getMessageId().getAnchorsToIds().keySet()) { TasksCommon.put_xor(anchors_to_ids, root_id, edge_id); } } } MessageId msgid=MessageId.makeId(anchors_to_ids); workerTransfer.transfer(t,new Tuple(topologyContext, values, task_id,out_stream_id, msgid)); } return StormUtils.mk_list(out_tasks); } catch (Exception e) { LOG.error("bolt emit", e); } return new ArrayList<Integer>(); }
MessageId msgId = tuple.getMessageId(); if (msgId != null) { msgId.serialize(output);
@Override public void fail(Tuple input) { // if ackerNum == 0, we can just return if (input.getMessageId() != null) { pending_acks.remove(input); for (Map.Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey()); sendBoltMsg(Acker.ACKER_FAIL_STREAM_ID, null, ackTuple, null, null); } } task_stats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId()); }
@Override public void fail(Tuple input) { // if ackerNum == 0, we can just return if (input.getMessageId() != null) { pending_acks.remove(input); for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) { unanchoredSend(topologyContext, sendTargets, taskTransfer, Acker.ACKER_FAIL_STREAM_ID, JStormUtils.mk_list((Object) e.getKey())); } } tuple_start_times.remove(input); task_stats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId()); }
@Override public void execute(Tuple tuple) { String sentence = tuple.getString(0); for (String word : sentence.split("\\s+")) { //锚定在这个位置 collector.emit(tuple,new Values(word)); } LOG.info("SPLIT_SENTENCE:{} msgId:{}",sentence, tuple.getMessageId().getAnchors()); collector.ack(tuple); }
@Override public String format(Tuple tuple) { Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); String values = ""; for (int i=0; i<tuple.size(); i++) { if (i != 0) values += ", "; values += String.format("%s=%s", schema.get(i), tuple.getValue(i)); } return String.format(TEMPLATE, tuple.getSourceComponent(), tuple.getSourceTask(), tuple.getSourceStreamId(), tuple.getMessageId().toString(), values); }
public byte[] serialize(Tuple tuple) { try { _kryoOut.clear(); _kryoOut.writeInt(tuple.getSourceTask(), true); _kryoOut.writeInt(_ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), true); tuple.getMessageId().serialize(_kryoOut); _kryo.serializeInto(tuple.getValues(), _kryoOut); return _kryoOut.toBytes(); } catch (IOException e) { throw new RuntimeException(e); } }