public static void send(TopologyContext topologyContext, TaskSendTargets taskTargets, TaskTransfer transferFn, String stream, List<Object> values) { List<Integer> tasks = taskTargets.get(stream, values, null, values.get(0)); if (tasks.size() == 0) { return; } Integer taskId = topologyContext.getThisTaskId(); for (Integer task : tasks) { TupleImplExt tup = new TupleImplExt(topologyContext, values, taskId, stream); tup.setTargetTaskId(task); transferFn.transfer(tup); } } }
@Override public void updateObject() { this.object = new TupleImplExt(topologyContext, new Values(TimeUtils.current_time_secs()), (int) Constants.SYSTEM_TASK_ID, Constants.SYSTEM_TICK_STREAM_ID); }
batchValues.add(pair); TupleImplExt batchTuple = new TupleImplExt(topology_context, batchValues, task_id, outStreamId, null); batchTuple.setTargetTaskId(t); batchTuple.setBatchTuple(true);
private void processTupleBatchEvent(Tuple tuple) { try { if ((!isSystemBolt && tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID)) || tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_REGISTER_METRICS_RESP_STREAM_ID)) { if (tuple.getValues().get(0) instanceof Pair) { for (Object value : tuple.getValues()) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; TupleImplExt t = new TupleImplExt( sysTopologyCtx, val.getSecond(), val.getFirst(), ((TupleImplExt) tuple)); processTupleEvent(t); } } } else { bolt.execute(tuple); } } catch (Throwable e) { error = e; LOG.error("bolt execute error ", e); reportError.report(e); } }
TupleImplExt tuple = new TupleImplExt(_context, values, taskId, streamName, id); tuple.setBatchTuple(isBatchTuple); tuple.setTargetTaskId(targetTaskId);
@SuppressWarnings("unchecked") public void sendHbMsg() { if (componentId.equals(Common.TOPOLOGY_MASTER_COMPONENT_ID)) { Values values = new Values(uptime.uptime(), executorStatus.getStatus()); TupleExt tuple = new TupleImplExt(sysTopologyCtx, values, taskId, Common.TOPOLOGY_MASTER_HB_STREAM_ID); queue.publish(tuple); } else { // Send task heartbeat to topology master List values = JStormUtils.mk_list(uptime.uptime(), executorStatus.getStatus()); LOG.debug("Send TaskHeartbeat from task:{}, uptime:{}", taskId, uptime.uptime()); if (spoutCollector != null) { spoutCollector.emitCtrl(Common.TOPOLOGY_MASTER_HB_STREAM_ID, values, null); } else if (boltCollector != null) { boltCollector.emitCtrl(Common.TOPOLOGY_MASTER_HB_STREAM_ID, null, values); } else { LOG.warn("Failed to send heartbeat msg: OutputCollector has not been initialized!"); } } }
for (Object value : values) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; TupleImplExt tuple = new TupleImplExt( sysTopologyCtx, val.getSecond(), val.getFirst(), ((TupleImplExt) event)); runnable = processTupleEvent(tuple);
TupleImplExt batchTuple = new TupleImplExt(topologyContext, batchValues, taskId, outStreamId, null); batchTuple.setTargetTaskId(t); batchTuple.setBatchTuple(true);
protected void processBatchTuple(Tuple batchEvent) { String stream = batchEvent.getSourceStreamId(); if (stream.equals(TransactionCommon.BARRIER_STREAM_ID)) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) batchEvent.getValue(0); BatchSnapshot snapshot = (BatchSnapshot) val.getSecond().get(0); if (currentBatchTracker.receiveBarrier(batchEvent.getSourceTask())) currentBatchTracker.expectedTupleCount += snapshot.getTupleCount(); else LOG.warn("Received expired or unexpected barrier={} from task-{}", snapshot, batchEvent.getSourceTask()); // LOG.debug("Received batch, stream={}, batchId={}, sourceTask={}, values={}", stream, currentBatchTracker.bactchId, // batchEvent.getSourceTask(), snapshot); LOG.debug("currentBatchTracker={}\n processingBatches={}\n PendingBatches={}\n intraPendingBatches={}", currentBatchTracker, processingBatches, batchCache, intraBatchCache); /* * if (currentBatchTracker.isAllBarriersReceived()) { if (currentBatchTracker.checkFinish()) finishCurrentBatch(); else * outputCollector.fail(batchEvent); } */ } else { for (Object value : batchEvent.getValues()) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; TupleImplExt tuple = new TupleImplExt(topologyContext, val.getSecond(), val.getFirst(), ((TupleImplExt) batchEvent)); boltExecutor.execute(tuple); } currentBatchTracker.incrementReceivedCount(batchEvent.getValues().size()); } if (currentBatchTracker.checkFinish()) { finishCurrentBatch(); } }
public List<Integer> sendMsg(String out_stream_id, List<Object> values, Collection<Tuple> anchors, Integer out_task_id, ICollectorCallback callback) { final long start = emitTimer.getTime(); List<Integer> outTasks = null; try { if (out_task_id != null) { outTasks = sendTargets.get(out_task_id, out_stream_id, values, anchors, null); } else { outTasks = sendTargets.get(out_stream_id, values, anchors, null); } tryRotate(); for (Integer t : outTasks) { MessageId msgId = getMessageId(anchors); TupleImplExt tp = new TupleImplExt(topologyContext, values, taskId, out_stream_id, msgId); tp.setTargetTaskId(t); taskTransfer.transfer(tp); } } catch (Exception e) { LOG.error("bolt emit error:", e); } finally { if (outTasks == null) { outTasks = new ArrayList<>(); } if (callback != null) callback.execute(out_stream_id, outTasks, values); emitTimer.updateTime(start); } return outTasks; }
protected List<Integer> sendCtrlMsg(String out_stream_id, List<Object> values, Collection<Tuple> anchors, Integer out_task_id) { final long start = emitTimer.getTime(); java.util.List<Integer> out_tasks = null; try { if (out_task_id != null) { out_tasks = sendTargets.get(out_task_id, out_stream_id, values, anchors, null); } else { out_tasks = sendTargets.get(out_stream_id, values, anchors, null); } tryRotate(); for (Integer t : out_tasks) { MessageId msgId = getMessageId(anchors); TupleImplExt tp = new TupleImplExt(topologyContext, values, taskId, out_stream_id, msgId); tp.setTargetTaskId(t); transferCtr(tp); } } catch (Exception e) { LOG.error("bolt emit error:", e); } finally { emitTimer.updateTime(start); } return out_tasks; }
TupleImplExt tp = new TupleImplExt(topology_context, values, task_id, out_stream_id, msgId); tp.setTargetTaskId(t); transferCtr(tp);
TupleImplExt tp = new TupleImplExt(topology_context, values, task_id, out_stream_id, msgId); tp.setTargetTaskId(t); transfer_fn.transfer(tp);
for (Object value : values) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; TupleImplExt t = new TupleImplExt( sysTopologyCtx, val.getSecond(), val.getFirst(), ((TupleImplExt) event)); processTupleEvent(t);
public static void send(TopologyContext topologyContext, TaskSendTargets taskTargets, TaskTransfer transfer_fn, String stream, List<Object> values) { List<Integer> tasks = taskTargets.get(stream, values, null, values.get(0)); if (tasks.size() == 0) { return; } Integer taskId = topologyContext.getThisTaskId(); for (Integer task : tasks) { TupleImplExt tup = new TupleImplExt(topologyContext, values, taskId, stream); tup.setTargetTaskId(task); transfer_fn.transfer(tup); } } }
@Override public void updateObject() { this.object = new TupleImplExt(topologyContext, new Values(TimeUtils.current_time_secs()), (int) Constants.SYSTEM_TASK_ID, Constants.SYSTEM_TICK_STREAM_ID); }
private void processTupleBatchEvent(Tuple tuple) { try { if ((!isSystemBolt && tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID)) || tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_REGISTER_METRICS_RESP_STREAM_ID)) { if (tuple.getValues().get(0) instanceof Pair) { for (Object value : tuple.getValues()) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; TupleImplExt t = new TupleImplExt(sysTopologyCtx, val.getSecond(), val.getFirst(), ((TupleImplExt) tuple)); processTupleEvent(t); } } } else { bolt.execute(tuple); } } catch (Throwable e) { error = e; LOG.error("bolt execute error ", e); report_error.report(e); } }
public void sendHbMsg() { if (componentId.equals(Common.TOPOLOGY_MASTER_COMPONENT_ID)) { Values values = new Values(uptime.uptime(), executorStatus.getStatus()); TupleExt tuple = new TupleImplExt(sysTopologyCtx, values, taskId, Common.TOPOLOGY_MASTER_HB_STREAM_ID); queue.publish(tuple); } else { // Send task heartbeat to topology master List values = JStormUtils.mk_list(uptime.uptime(), executorStatus.getStatus()); if (spoutCollector != null) { spoutCollector.emitCtrl(Common.TOPOLOGY_MASTER_HB_STREAM_ID, values, null); } else if (boltCollector != null) { boltCollector.emitCtrl(Common.TOPOLOGY_MASTER_HB_STREAM_ID, null, values); } else { LOG.warn("Failed to send hearbeat msg. OutputCollector has not been initialized!"); } } }
public void processBatchTuple(Tuple batchEvent) { String stream = batchEvent.getSourceStreamId(); if (stream.equals(TransactionCommon.BARRIER_STREAM_ID)) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) batchEvent.getValue(0); BatchSnapshot snapshot = (BatchSnapshot) val.getSecond().get(1); currentBatchTracker.receiveBarrier(batchEvent.getSourceTask()); currentBatchTracker.expectedTupleCount += snapshot.getTupleCount(); LOG.debug("Received batch, stream={}, batchGroupId={}, sourceTask={}, values={}", stream, currentBatchTracker.bactchGroupId, batchEvent.getSourceTask(), snapshot); LOG.debug("currentBatchTracker={}, processingBatches={}, pendingBatches={}", currentBatchTracker, processingBatches, batchCache); } else { for (Object value : batchEvent.getValues()) { /*List<Object> firstTupleValue = ((Pair<MessageId, List<Object>>) value).getSecond(); BatchGroupId batchGroupId = (BatchGroupId) firstTupleValue.get(0); if (!batchGroupId.equals(currentBatchTracker.bactchGroupId)) { LOG.warn("batchgroupid-{} is not equal to the once of current batch tracker-{}!", batchGroupId, currentBatchTracker.bactchGroupId); }*/ Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; val.getSecond().remove(0); TupleImplExt tuple = new TupleImplExt(topologyContext, val.getSecond(), val.getFirst(), ((TupleImplExt) batchEvent)); boltExecutor.execute(tuple); } currentBatchTracker.incrementReceivedCount(batchEvent.getValues().size()); } if (currentBatchTracker.checkFinish()) { finishCurrentBatch(); } }
protected List<Integer> sendCtrlMsg(String out_stream_id, List<Object> values, Collection<Tuple> anchors, Integer out_task_id) { final long start = emitTimer.getTime(); java.util.List<Integer> out_tasks = null; try { if (out_task_id != null) { out_tasks = sendTargets.get(out_task_id, out_stream_id, values, anchors, null); } else { out_tasks = sendTargets.get(out_stream_id, values, anchors, null); } tryRotate(); for (Integer t : out_tasks) { MessageId msgid = getMessageId(anchors); TupleImplExt tp = new TupleImplExt(topologyContext, values, task_id, out_stream_id, msgid); tp.setTargetTaskId(t); transferCtr(tp); } } catch (Exception e) { LOG.error("bolt emit", e); } finally { emitTimer.updateTime(start); } return out_tasks; }