@Override public String getStream() { return stream.get_streamId(); } });
public java.lang.Object getFieldValue(_Fields field) { switch (field) { case COMPONENT_ID: return get_componentId(); case STREAM_ID: return get_streamId(); } throw new java.lang.IllegalStateException(); }
private Set<GlobalStreamId> getComponentStreams(TopologyContext context) { Set<GlobalStreamId> streams = new HashSet<>(); for (GlobalStreamId streamId : context.getThisSources().keySet()) { if (!streamId.get_streamId().equals(CheckpointSpout.CHECKPOINT_STREAM_ID)) { streams.add(streamId); } } return streams; }
private String getStreamId(GlobalStreamId globalStreamId) { if (globalStreamId.get_componentId().startsWith("spout")) { return globalStreamId.get_componentId() + globalStreamId.get_streamId(); } return globalStreamId.get_streamId(); }
/** * Remap the names of components. * @param remappedComponents old name to new name of components. * @param remappedStreams old ID to new ID of streams. * @return a modified version of this with names remapped. */ public InputStream remap(Map<String, String> remappedComponents, Map<GlobalStreamId, GlobalStreamId> remappedStreams) { String remapTo = remappedComponents.get(toComponent); String remapFrom = remappedComponents.get(fromComponent); GlobalStreamId remapStreamId = remappedStreams.get(gsid()); return new InputStream(remapFrom, remapTo, remapStreamId.get_streamId(), execTime, processTime, groupingType); }
/** * getInputMap. * @param entryInput entryInput * @return getInputMap */ public static Map<String, Object> getInputMap(Map.Entry<GlobalStreamId,Grouping> entryInput) { Map<String, Object> result = new HashMap(); result.put(":component", entryInput.getKey().get_componentId()); result.put(":stream", entryInput.getKey().get_streamId()); result.put(":sani-stream", sanitizeStreamName(entryInput.getKey().get_streamId())); result.put(":grouping", entryInput.getValue().getSetField().getFieldName()); return result; }
private Fields getSourceOutputFields(TopologyContext context, String sourceStream) { for (GlobalStreamId g : context.getThisSources().keySet()) { if (g.get_streamId().equals(sourceStream)) { return context.getComponentOutputFields(g); } } throw new RuntimeException("Could not find fields for source stream " + sourceStream); }
public OutputStream remap(String origId, Map<GlobalStreamId, GlobalStreamId> remappedStreams) { GlobalStreamId remapped = remappedStreams.get(new GlobalStreamId(origId, id)); return new OutputStream(remapped.get_streamId(), rate, areKeysSkewed); }
/** * returns the total number of input checkpoint streams across all input tasks to this component. */ private int getCheckpointInputTaskCount(TopologyContext context) { int count = 0; for (GlobalStreamId inputStream : context.getThisSources().keySet()) { if (CHECKPOINT_STREAM_ID.equals(inputStream.get_streamId())) { count += context.getComponentTasks(inputStream.get_componentId()).size(); } } return count; }
@Override public BoltDeclarer grouping(GlobalStreamId id, Grouping grouping) { return grouping(id.get_componentId(), id.get_streamId(), grouping); } }
/** * Gets the declared output fields for the specified global stream id. */ public Fields getComponentOutputFields(GlobalStreamId id) { return getComponentOutputFields(id.get_componentId(), id.get_streamId()); }
/** * Gets information about who is consuming the outputs of the specified component, and how. * * @return Map from stream id to component id to the Grouping used. */ public Map<String, Map<String, Grouping>> getTargets(String componentId) { Map<String, Map<String, Grouping>> ret = new HashMap<>(); for (String otherComponentId : getComponentIds()) { Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs(); for (Map.Entry<GlobalStreamId, Grouping> entry : inputs.entrySet()) { GlobalStreamId id = entry.getKey(); if (id.get_componentId().equals(componentId)) { Map<String, Grouping> curr = ret.get(id.get_streamId()); if (curr == null) { curr = new HashMap<>(); } curr.put(otherComponentId, entry.getValue()); ret.put(id.get_streamId(), curr); } } } return ret; }
@Override public List<String> transform(Object key) { GlobalStreamId sid = (GlobalStreamId) key; return Lists.newArrayList(sid.get_componentId(), sid.get_streamId()); } }
@Override public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) { _fieldLocations = new HashMap<String, GlobalStreamId>(); _collector = collector; int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback()); _numSources = context.getThisSources().size(); Set<String> idFields = null; for (GlobalStreamId source : context.getThisSources().keySet()) { Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); Set<String> setFields = new HashSet<String>(fields.toList()); if (idFields == null) { idFields = setFields; } else { idFields.retainAll(setFields); } for (String outfield : _outFields) { for (String sourcefield : fields) { if (outfield.equals(sourcefield)) { _fieldLocations.put(outfield, source); } } } } _idFields = new Fields(new ArrayList<String>(idFields)); if (_fieldLocations.size() != _outFields.size()) { throw new RuntimeException("Cannot find all outfields among sources"); } }
/** * getBoltInputStats. * @param globalStreamId globalStreamId * @param componentAggregateStats componentAggregateStats * @return getBoltInputStats */ private static Map<String, Object> getBoltInputStats(GlobalStreamId globalStreamId, ComponentAggregateStats componentAggregateStats) { Map<String, Object> result = new HashMap(); SpecificAggregateStats specificAggregateStats = componentAggregateStats.get_specific_stats(); BoltAggregateStats boltAggregateStats = specificAggregateStats.get_bolt(); CommonAggregateStats commonAggregateStats = componentAggregateStats.get_common_stats(); String componentId = globalStreamId.get_componentId(); result.put("component", componentId); result.put("encodedComponentId", URLEncoder.encode(componentId)); result.put("stream", globalStreamId.get_streamId()); result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats.get_execute_latency_ms())); result.put("processLatency", StatsUtil.floatStr(boltAggregateStats.get_process_latency_ms())); result.put("executed", nullToZero(boltAggregateStats.get_executed())); result.put("acked", nullToZero(commonAggregateStats.get_acked())); result.put("failed", nullToZero(commonAggregateStats.get_failed())); return result; }
private void setUpProcessorBolt(Processor<?> processor, Set<String> windowedParentStreams, boolean isWindowed, String tsFieldName) { ProcessorNode node = new ProcessorNode(processor, "outputstream", new Fields("value")); node.setWindowedParentStreams(windowedParentStreams); node.setWindowed(isWindowed); Mockito.when(mockStreamToProcessors.get(Mockito.anyString())).thenReturn(Collections.singletonList(node)); Mockito.when(mockStreamToProcessors.keySet()).thenReturn(Collections.singleton("inputstream")); Map<GlobalStreamId, Grouping> mockSources = Mockito.mock(Map.class); GlobalStreamId mockGlobalStreamId = Mockito.mock(GlobalStreamId.class); Mockito.when(mockTopologyContext.getThisSources()).thenReturn(mockSources); Mockito.when(mockSources.keySet()).thenReturn(Collections.singleton(mockGlobalStreamId)); Mockito.when(mockGlobalStreamId.get_streamId()).thenReturn("inputstream"); Mockito.when(mockGlobalStreamId.get_componentId()).thenReturn("bolt0"); Mockito.when(mockTopologyContext.getComponentTasks(Mockito.anyString())).thenReturn(Collections.singletonList(1)); graph.addVertex(node); bolt = new ProcessorBolt("bolt1", graph, Collections.singletonList(node)); if (tsFieldName != null && !tsFieldName.isEmpty()) { bolt.setTimestampField(tsFieldName); } bolt.setStreamToInitialProcessors(mockStreamToProcessors); bolt.prepare(new HashMap<>(), mockTopologyContext, mockOutputCollector); }
@Test public void testDeserialize_readingFromGzip() throws Exception { GlobalStreamId id = new GlobalStreamId("first", "second"); byte[] serialized = new GzipThriftSerializationDelegate().serialize(id); GlobalStreamId id2 = testDelegate.deserialize(serialized, GlobalStreamId.class); assertEquals(id2.get_componentId(), id.get_componentId()); assertEquals(id2.get_streamId(), id.get_streamId()); }
Pair<Object, String> getValueAndStream(Tuple input) { Object value; String stream; // if tuple arrives from a spout, it can be passed as is // otherwise the value is in the first field of the tuple if (input.getSourceComponent().startsWith("spout")) { value = input; stream = input.getSourceGlobalStreamId().get_componentId() + input.getSourceGlobalStreamId().get_streamId(); } else if (isPair(input)) { value = Pair.of(input.getValue(0), input.getValue(1)); stream = input.getSourceStreamId(); } else { value = input.getValue(0); stream = input.getSourceStreamId(); } return Pair.of(value, stream); }
@Test public void testDeserialize_readingFromGzipBridge() throws Exception { GlobalStreamId id = new GlobalStreamId("first", "second"); byte[] serialized = new GzipBridgeThriftSerializationDelegate().serialize(id); GlobalStreamId id2 = testDelegate.deserialize(serialized, GlobalStreamId.class); assertEquals(id2.get_componentId(), id.get_componentId()); assertEquals(id2.get_streamId(), id.get_streamId()); }
@Test public void testDeserialize_readingFromDefault() throws Exception { GlobalStreamId id = new GlobalStreamId("A", "B"); byte[] serialized = new ThriftSerializationDelegate().serialize(id); GlobalStreamId id2 = testDelegate.deserialize(serialized, GlobalStreamId.class); assertEquals(id2.get_componentId(), id.get_componentId()); assertEquals(id2.get_streamId(), id.get_streamId()); } }