@Override public String getComponent() { return stream.get_componentId(); } });
@Override public String getComponent() { return stream.get_componentId(); }
Map<String, Set<String>> getBoltBatchToComponentSubscriptions(String id) { Map<String, Set<String>> ret = new HashMap<>(); for (GlobalStreamId s : getBoltSubscriptionStreams(id)) { String b = batchIds.get(s); if (!ret.containsKey(b)) { ret.put(b, new HashSet<>()); } ret.get(b).add(s.get_componentId()); } return ret; }
private String getStreamId(GlobalStreamId globalStreamId) { if (globalStreamId.get_componentId().startsWith("spout")) { return globalStreamId.get_componentId() + globalStreamId.get_streamId(); } return globalStreamId.get_streamId(); }
private Set<String> getInputsTo(ComponentCommon comp) { Set<String> ret = new HashSet<>(); for (GlobalStreamId globalId : comp.get_inputs().keySet()) { ret.add(globalId.get_componentId()); } return ret; }
public java.lang.Object getFieldValue(_Fields field) { switch (field) { case COMPONENT_ID: return get_componentId(); case STREAM_ID: return get_streamId(); } throw new java.lang.IllegalStateException(); }
@Override public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> tasks) { List<Integer> sourceTasks = new ArrayList<>(context.getComponentTasks(stream.get_componentId())); Collections.sort(sourceTasks); if (sourceTasks.size() != tasks.size()) { throw new RuntimeException("Can only do an identity grouping when source and target have same number of tasks"); } tasks = new ArrayList<>(tasks); Collections.sort(tasks); for (int i = 0; i < sourceTasks.size(); i++) { int s = sourceTasks.get(i); int t = tasks.get(i); _precomputed.put(s, Arrays.asList(t)); } }
private int getStreamInputTaskCount(TopologyContext context, String stream) { int count = 0; for (GlobalStreamId inputStream : context.getThisSources().keySet()) { if (stream.equals(getStreamId(inputStream))) { count += context.getComponentTasks(inputStream.get_componentId()).size(); } } return count; }
/** * returns the total number of input checkpoint streams across all input tasks to this component. */ private int getCheckpointInputTaskCount(TopologyContext context) { int count = 0; for (GlobalStreamId inputStream : context.getThisSources().keySet()) { if (CHECKPOINT_STREAM_ID.equals(inputStream.get_streamId())) { count += context.getComponentTasks(inputStream.get_componentId()).size(); } } return count; }
/** * getInputMap. * @param entryInput entryInput * @return getInputMap */ public static Map<String, Object> getInputMap(Map.Entry<GlobalStreamId,Grouping> entryInput) { Map<String, Object> result = new HashMap(); result.put(":component", entryInput.getKey().get_componentId()); result.put(":stream", entryInput.getKey().get_streamId()); result.put(":sani-stream", sanitizeStreamName(entryInput.getKey().get_streamId())); result.put(":grouping", entryInput.getValue().getSetField().getFieldName()); return result; }
/** * Gets the declared output fields for the specified global stream id. */ public Fields getComponentOutputFields(GlobalStreamId id) { return getComponentOutputFields(id.get_componentId(), id.get_streamId()); }
@Override public BoltDeclarer grouping(GlobalStreamId id, Grouping grouping) { return grouping(id.get_componentId(), id.get_streamId(), grouping); } }
@Override public List<String> transform(Object key) { GlobalStreamId sid = (GlobalStreamId) key; return Lists.newArrayList(sid.get_componentId(), sid.get_streamId()); } }
/** * Gets the declared input fields for this component. * * @return A map from sources to streams to fields. */ public Map<String, Map<String, List<String>>> getThisInputFields() { Map<String, Map<String, List<String>>> outputMap = new HashMap<>(); for (Map.Entry<GlobalStreamId, Grouping> entry : this.getThisSources().entrySet()) { String componentId = entry.getKey().get_componentId(); Set<String> streams = getComponentStreams(componentId); for (String stream : streams) { Map<String, List<String>> streamFieldMap = outputMap.get(componentId); if (streamFieldMap == null) { streamFieldMap = new HashMap<>(); outputMap.put(componentId, streamFieldMap); } streamFieldMap.put(stream, getComponentOutputFields(componentId, stream).toList()); } } return outputMap; }
/** * For bolts that has incoming streams from spouts (the root bolts), add checkpoint stream from checkpoint spout to its input. For other * bolts, add checkpoint stream from the previous bolt to its input. */ private void addCheckPointInputs(ComponentCommon component) { Set<GlobalStreamId> checkPointInputs = new HashSet<>(); for (GlobalStreamId inputStream : component.get_inputs().keySet()) { String sourceId = inputStream.get_componentId(); if (_spouts.containsKey(sourceId)) { checkPointInputs.add(new GlobalStreamId(CHECKPOINT_COMPONENT_ID, CHECKPOINT_STREAM_ID)); } else { checkPointInputs.add(new GlobalStreamId(sourceId, CHECKPOINT_STREAM_ID)); } } for (GlobalStreamId streamId : checkPointInputs) { component.put_to_inputs(streamId, Grouping.all(new NullStruct())); } }
@Override public void initState(T state) { if (stateInitialized) { LOG.warn("State is already initialized. Ignoring initState"); return; } statefulWindowedBolt.initState((T) state); // query the streamState for each input task stream and compute recoveryStates for (GlobalStreamId streamId : topologyContext.getThisSources().keySet()) { for (int taskId : topologyContext.getComponentTasks(streamId.get_componentId())) { WindowState windowState = streamState.get(new TaskStream(taskId, streamId)); if (windowState != null) { recoveryStates.put(new TaskStream(taskId, streamId), windowState); } } } LOG.debug("recoveryStates {}", recoveryStates); stateInitialized = true; start(); }
Pair<Object, String> getValueAndStream(Tuple input) { Object value; String stream; // if tuple arrives from a spout, it can be passed as is // otherwise the value is in the first field of the tuple if (input.getSourceComponent().startsWith("spout")) { value = input; stream = input.getSourceGlobalStreamId().get_componentId() + input.getSourceGlobalStreamId().get_streamId(); } else if (isPair(input)) { value = Pair.of(input.getValue(0), input.getValue(1)); stream = input.getSourceStreamId(); } else { value = input.getValue(0); stream = input.getSourceStreamId(); } return Pair.of(value, stream); }
@Test public void testDeserialize_readingFromGzipBridge() throws Exception { GlobalStreamId id = new GlobalStreamId("first", "second"); byte[] serialized = new GzipBridgeThriftSerializationDelegate().serialize(id); GlobalStreamId id2 = testDelegate.deserialize(serialized, GlobalStreamId.class); assertEquals(id2.get_componentId(), id.get_componentId()); assertEquals(id2.get_streamId(), id.get_streamId()); }
@Test public void testDeserialize_readingFromDefault() throws Exception { GlobalStreamId id = new GlobalStreamId("A", "B"); byte[] serialized = new ThriftSerializationDelegate().serialize(id); GlobalStreamId id2 = testDelegate.deserialize(serialized, GlobalStreamId.class); assertEquals(id2.get_componentId(), id.get_componentId()); assertEquals(id2.get_streamId(), id.get_streamId()); } }
@Test public void testDeserialize_readingFromGzip() throws Exception { GlobalStreamId id = new GlobalStreamId("first", "second"); byte[] serialized = new GzipThriftSerializationDelegate().serialize(id); GlobalStreamId id2 = testDelegate.deserialize(serialized, GlobalStreamId.class); assertEquals(id2.get_componentId(), id.get_componentId()); assertEquals(id2.get_streamId(), id.get_streamId()); }