void addOutEdge(StreamEdge out) { outEdges.put(out.getStreamSpec().getId(), out); }
void addInEdge(StreamEdge in) { inEdges.put(in.getStreamSpec().getId(), in); }
void addOutEdge(StreamEdge out) { outEdges.put(out.getStreamSpec().getId(), out); }
void addInEdge(StreamEdge in) { inEdges.put(in.getStreamSpec().getId(), in); }
void addOutEdge(StreamEdge out) { outEdges.put(out.getStreamSpec().getId(), out); }
void addOutEdge(StreamEdge out) { outEdges.put(out.getStreamSpec().getId(), out); }
void addInEdge(StreamEdge in) { inEdges.put(in.getStreamSpec().getId(), in); }
void addOutEdge(StreamEdge out) { outEdges.put(out.getStreamSpec().getId(), out); }
Collection<OperatorSpec> getReachableOperators() { Set<OperatorSpec> inputOperatorsInJobNode = inEdges.values().stream().map(inEdge -> appDesc.getInputOperators().get(inEdge.getStreamSpec().getId())).filter(Objects::nonNull).collect(Collectors.toSet()); Set<OperatorSpec> reachableOperators = new HashSet<>(); findReachableOperators(inputOperatorsInJobNode, reachableOperators); return reachableOperators; }
Collection<OperatorSpec> getReachableOperators() { Set<OperatorSpec> inputOperatorsInJobNode = inEdges.values().stream().map(inEdge -> appDesc.getInputOperators().get(inEdge.getStreamSpec().getId())).filter(Objects::nonNull).collect(Collectors.toSet()); Set<OperatorSpec> reachableOperators = new HashSet<>(); findReachableOperators(inputOperatorsInJobNode, reachableOperators); return reachableOperators; }
Collection<OperatorSpec> getReachableOperators() { Set<OperatorSpec> inputOperatorsInJobNode = inEdges.values().stream().map(inEdge -> appDesc.getInputOperators().get(inEdge.getStreamSpec().getId())).filter(Objects::nonNull).collect(Collectors.toSet()); Set<OperatorSpec> reachableOperators = new HashSet<>(); findReachableOperators(inputOperatorsInJobNode, reachableOperators); return reachableOperators; }
Collection<OperatorSpec> getReachableOperators() { Set<OperatorSpec> inputOperatorsInJobNode = inEdges.values().stream().map(inEdge -> appDesc.getInputOperators().get(inEdge.getStreamSpec().getId())).filter(Objects::nonNull).collect(Collectors.toSet()); Set<OperatorSpec> reachableOperators = new HashSet<>(); findReachableOperators(inputOperatorsInJobNode, reachableOperators); return reachableOperators; }
Collection<OperatorSpec> getReachableOperators() { Set<OperatorSpec> inputOperatorsInJobNode = inEdges.values().stream().map(inEdge -> appDesc.getInputOperators().get(inEdge.getStreamSpec().getId())).filter(Objects::nonNull).collect(Collectors.toSet()); Set<OperatorSpec> reachableOperators = new HashSet<>(); findReachableOperators(inputOperatorsInJobNode, reachableOperators); return reachableOperators; }
Config generateConfig() { Map<String, String> streamConfig = new HashMap<>(); StreamSpec spec = getStreamSpec(); streamConfig.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), spec.getId()), spec.getSystemName()); streamConfig.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), spec.getId()), spec.getPhysicalName()); if (isIntermediate()) { streamConfig.put(String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), spec.getId()), "true"); streamConfig.put(String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), spec.getId()), "true"); streamConfig.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), spec.getId()), "oldest"); streamConfig.put(String.format(StreamConfig.PRIORITY_FOR_STREAM_ID(), spec.getId()), String.valueOf(Integer.MAX_VALUE)); } spec.getConfig().forEach((property, value) -> { streamConfig.put(String.format(StreamConfig.STREAM_ID_PREFIX(), spec.getId()) + property, value); }); return new MapConfig(streamConfig); }
Config generateConfig() { Map<String, String> config = new HashMap<>(); StreamSpec spec = getStreamSpec(); config.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), spec.getId()), spec.getSystemName()); config.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), spec.getId()), spec.getPhysicalName()); if (isIntermediate()) { config.put(String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), spec.getId()), "true"); config.put(String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), spec.getId()), "true"); config.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), spec.getId()), "oldest"); config.put(String.format(StreamConfig.PRIORITY_FOR_STREAM_ID(), spec.getId()), String.valueOf(Integer.MAX_VALUE)); } spec.getConfig().forEach((property, value) -> { config.put(String.format(StreamConfig.STREAM_ID_PREFIX(), spec.getId()) + property, value); }); return new MapConfig(config); }
/** * Get the {@link StreamEdge} for a {@link StreamSpec}. Create one if it does not exist. * @param streamSpec spec of the StreamEdge * @param isIntermediate boolean flag indicating whether it's an intermediate stream * @return stream edge */ private StreamEdge getOrCreateStreamEdge(StreamSpec streamSpec, boolean isIntermediate) { String streamId = streamSpec.getId(); StreamEdge edge = edges.get(streamId); if (edge == null) { boolean isBroadcast = appDesc.getIntermediateBroadcastStreamIds().contains(streamId); edge = new StreamEdge(streamSpec, isIntermediate, isBroadcast, config); edges.put(streamId, edge); } return edge; }
@Test public void testGetStreamSystemNameArgValid() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg StreamConfig.SYSTEM(), TEST_SYSTEM); // This too StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); assertEquals(TEST_SYSTEM, spec.getSystemName()); }
@Test public void testGetStreamPhysicalNameArgSimple() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg StreamConfig.SYSTEM(), TEST_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME2, spec.getPhysicalName()); assertEquals(TEST_SYSTEM, spec.getSystemName()); }
@Test public void testBasicConstructor() { StreamSpec streamSpec = new StreamSpec("dummyId", "dummyPhysicalName", "dummySystemName", 1); assertEquals("dummyId", streamSpec.getId()); assertEquals("dummyPhysicalName", streamSpec.getPhysicalName()); assertEquals("dummySystemName", streamSpec.getSystemName()); assertEquals(1, streamSpec.getPartitionCount()); // SystemStream should use the physical name, not the streamId. SystemStream systemStream = new SystemStream("dummySystemName", "dummyPhysicalName"); assertEquals(systemStream, streamSpec.toSystemStream()); }