public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _tracker = new SpoutTrackOutputCollector(collector); _delegate.open(conf, context, new SpoutOutputCollector(_tracker)); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { super.open(conf, context, collector); spoutExecutor.open(conf, context, new SpoutOutputCollector(outputCollector)); int threadPoolNum = JStormUtils.parseInt(conf.get("transaction.schedule.thread.pool"), 1); int delay = JStormUtils.parseInt(conf.get("transaction.schedule.batch.delay.ms"), 1000); int initDelay = delay >= 30000 ? 30000 : delay; if (scheduledService == null) { scheduledService = Executors.newScheduledThreadPool(threadPoolNum); } scheduledService.scheduleWithFixedDelay(new Runnable() { @Override public void run() { process(Operation.commit, null); } }, initDelay, delay, TimeUnit.MILLISECONDS); } }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _delegate.open(conf, context, new SpoutOutputCollector(new StreamOverrideCollector(collector))); _outputTasks = new ArrayList<>(); for(String component: Utils.get(context.getThisTargets(), _coordStream, new HashMap<String, Grouping>()).keySet()) { _outputTasks.addAll(context.getComponentTasks(component)); } _rand = new Random(Utils.secureRandomLong()); }
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _tracker = new SpoutTrackOutputCollector(collector); _delegate.open(conf, context, new SpoutOutputCollector(_tracker)); }
public SpoutExecutors(backtype.storm.spout.ISpout _spout, WorkerTransfer _transfer_fn, Map _storm_conf, IConnection _puller, TaskSendTargets sendTargets, AtomicBoolean _storm_active_atom, TopologyContext topology_context, TopologyContext _user_context, BaseTaskStatsRolling _task_stats) { this.spout = _spout; this.storm_conf = _storm_conf; this.puller = _puller; this.zkActive = _storm_active_atom; this.user_context = _user_context; this.task_stats = _task_stats; Integer task_id = topology_context.getThisTaskId(); this.component_id = topology_context.getThisComponentId(); this.max_spout_pending = StormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING)); this.deserializer = new KryoTupleDeserializer(storm_conf,topology_context);// (KryoTupleDeserializer. // storm-conf this.event_queue = new ConcurrentLinkedQueue<Runnable>(); int message_timeout_secs =StormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); // شtupleĶУȴtuple̫spoutͣTimeCacheMaptuple�ʱtuple.fail֪ͨackerطtyple this.pending = new TimeCacheMap(message_timeout_secs,new SpoutTimeCallBack<Object, Object>(event_queue, spout,storm_conf,task_stats)); // collector,ʵǵsend_spout_msg this.output_collector = new SpoutCollector(task_id,spout,task_stats,sendTargets,storm_conf,_transfer_fn,pending,topology_context,event_queue); LOG.info("Opening spout " + component_id + ":" + task_id); this.spout.open(storm_conf, user_context, new SpoutOutputCollector(output_collector)); LOG.info("Opend spout " + component_id + ":" + task_id); this.isRecvRun=new AtomicBoolean(); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { SpoutOutputCollectorCb ackOutput = new AckSpoutOutputCollector(collector.getDelegate()); spoutExecutor.open(conf, context, new SpoutOutputCollector(ackOutput)); tracker = new AckPendingBatchTracker<>(); taskStats = new TaskBaseMetric(context.getTopologyId(), context.getThisComponentId(), context.getThisTaskId()); random = new Random(Utils.secureRandomLong()); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { super.open(conf, context, collector); /** * For backward compatibility to release 2.2.*, the conversion of output collector to * BasicSpoutOutputCollector is still kept here. */ if (spoutExecutor instanceof BasicTransactionSpoutExecutor) { ((BasicTransactionSpoutExecutor) spoutExecutor).open(conf, context, new BasicSpoutOutputCollector(outputCollector)); } else if (spoutExecutor instanceof IBasicTransactionSpoutExecutor) { ((IBasicTransactionSpoutExecutor) spoutExecutor).open(conf, context, new BasicSpoutOutputCollector(outputCollector)); } else { spoutExecutor.open(conf, context, new SpoutOutputCollector(outputCollector)); } } }
@Override public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) { long txid = tx.getTransactionId(); long now = System.currentTimeMillis(); if (now - lastRotate > rotateTime) { Map<Long, List<Object>> failed = idsMap.rotate(); for (Long id : failed.keySet()) { // TODO: this isn't right... it's not in the map anymore fail(id); } lastRotate = now; } if (idsMap.containsKey(txid)) { fail(txid); } _collector.reset(collector); if (!prepared) { _spout.open(_conf, _context, new SpoutOutputCollector(_collector)); prepared = true; } for (int i = 0; i < _maxBatchSize; i++) { _spout.nextTuple(); if(_collector.numEmitted < i) { break; } } idsMap.put(txid, _collector.ids); _collector.pendingCount = idsMap.size(); }
this.outputCollector = new SpoutOutputCollector(collector);
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _tracker = new SpoutTrackOutputCollector(collector); _delegate.open(conf, context, new SpoutOutputCollector(_tracker)); }
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _tracker = new SpoutTrackOutputCollector(collector); _delegate.open(conf, context, new SpoutOutputCollector(_tracker)); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _delegate.open(conf, context, new SpoutOutputCollector(new StreamOverrideCollector(collector))); _outputTasks = new ArrayList<Integer>(); for(String component: Utils.get(context.getThisTargets(), _coordStream, new HashMap<String, Grouping>()).keySet()) { _outputTasks.addAll(context.getComponentTasks(component)); } _rand = new Random(Utils.secureRandomLong()); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _delegate.open(conf, context, new SpoutOutputCollector(new StreamOverrideCollector(collector))); _outputTasks = new ArrayList<>(); for(String component: Utils.get(context.getThisTargets(), _coordStream, new HashMap<String, Grouping>()).keySet()) { _outputTasks.addAll(context.getComponentTasks(component)); } _rand = new Random(Utils.secureRandomLong()); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { this.conf = conf; this.topologyContext = context; this.topologyId = topologyContext.getTopologyId(); this.taskId = topologyContext.getThisTaskId(); this.topologyMasterId = topologyContext.getTopologyMasterId(); this.componentId = topologyContext.getThisComponentId(); this.groupId = TransactionCommon.groupIndex(context.getRawTopology(), componentId); this.downstreamTasks = TransactionCommon.getDownstreamTasks(componentId, topologyContext); this.outputCollector = new TransactionSpoutOutputCollector(collector, this); this.spoutStatus = State.INIT; this.committingBatches = new TreeSet<Long>(); this.isMaxPending = false; this.MAX_PENDING_BATCH_NUM = JStormUtils.parseInt(conf.get("transaction.max.pending.batch"), 2); int taskLaunchTimeout = JStormUtils.parseInt(conf.get(Config.NIMBUS_TASK_LAUNCH_SECS)); int spoutInitRetryDelaySec = JStormUtils.parseInt(conf.get("transaction.spout.init.retry.secs"), taskLaunchTimeout); this.initRetryCheck = new IntervalCheck(); initRetryCheck.setInterval(spoutInitRetryDelaySec); this.lock = new ReentrantLock(true); spoutExecutor.open(conf, context, new SpoutOutputCollector(outputCollector)); }
_spout.open(_conf, _context, new SpoutOutputCollector(_collector)); prepared = true;
@Override public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) { long txid = tx.getTransactionId(); long now = System.currentTimeMillis(); if (now - lastRotate > rotateTime) { Map<Long, List<Object>> failed = idsMap.rotate(); for (Long id : failed.keySet()) { // TODO: this isn't right... it's not in the map anymore fail(id); } lastRotate = now; } if (idsMap.containsKey(txid)) { fail(txid); } _collector.reset(collector); if (!prepared) { _spout.open(_conf, _context, new SpoutOutputCollector(_collector)); prepared = true; } for (int i = 0; i < _maxBatchSize; i++) { _spout.nextTuple(); if(_collector.numEmitted < i) { break; } } idsMap.put(txid, _collector.ids); _collector.pendingCount = idsMap.size(); }
this.outputCollector = new SpoutOutputCollector(collector);