public static Set<StreamSpec> getStreamSpecs(Set<String> streamIds, StreamConfig streamConfig) { return streamIds.stream().map(streamId -> getStreamSpec(streamId, streamConfig)).collect(Collectors.toSet()); }
private void configureTables(Map<String, String> generatedConfig, Config originalConfig, Map<String, TableSpec> tables, Set<String> inputs) { generatedConfig.putAll( TableConfigGenerator.generateConfigsForTableSpecs( new MapConfig(generatedConfig), new ArrayList<>(tables.values()))); // Add side inputs to the inputs and mark the stream as bootstrap tables.values().forEach(tableSpec -> { List<String> sideInputs = tableSpec.getSideInputs(); if (sideInputs != null && !sideInputs.isEmpty()) { sideInputs.stream() .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); generatedConfig.put(String.format(StreamConfig.STREAM_PREFIX() + StreamConfig.BOOTSTRAP(), systemStream.getSystem(), systemStream.getStream()), "true"); }); } }); }
String systemStreamName = systemStreamPartition.substring(0, hashPosition); String partitionSegment = systemStreamPartition.substring(hashPosition + 1); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(systemStreamName);
JobGraph jobGraph = new JobGraph(config, appDesc); Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig); Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig); jobGraph.addSideInputStream(getStreamSpec(sideInput, streamConfig));
StreamEdge(StreamSpec streamSpec, boolean isIntermediate, boolean isBroadcast, Config config) { this.streamSpec = streamSpec; this.isIntermediate = isIntermediate; this.isBroadcast = isBroadcast; this.config = config; if (isBroadcast) { partitions = 1; } this.name = StreamUtil.getNameFromSystemStream(getSystemStream()); }
JobGraph jobGraph = new JobGraph(config, appDesc); Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig); Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig); jobGraph.addSideInputStream(getStreamSpec(sideInput, streamConfig));
StreamEdge(StreamSpec streamSpec, boolean isIntermediate, boolean isBroadcast, Config config) { this.streamSpec = streamSpec; this.isIntermediate = isIntermediate; this.isBroadcast = isBroadcast; this.config = config; if (isBroadcast) { partitions = 1; } this.name = StreamUtil.getNameFromSystemStream(getSystemStream()); }
public static Set<StreamSpec> getStreamSpecs(Set<String> streamIds, StreamConfig streamConfig) { return streamIds.stream().map(streamId -> getStreamSpec(streamId, streamConfig)).collect(Collectors.toSet()); }
JobGraph jobGraph = new JobGraph(config, appDesc); Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig); Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig); jobGraph.addSideInputStream(getStreamSpec(sideInput, streamConfig));
private void configureTables(Map<String, String> generatedConfig, Config originalConfig, Map<String, TableDescriptor> tables, Set<String> inputs) { generatedConfig.putAll( TableConfigGenerator.generate( new MapConfig(generatedConfig), new ArrayList<>(tables.values()))); // Add side inputs to the inputs and mark the stream as bootstrap tables.values().forEach(tableDescriptor -> { if (tableDescriptor instanceof LocalTableDescriptor) { LocalTableDescriptor localTableDescriptor = (LocalTableDescriptor) tableDescriptor; List<String> sideInputs = localTableDescriptor.getSideInputs(); if (sideInputs != null && !sideInputs.isEmpty()) { sideInputs.stream() .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); generatedConfig.put(String.format(StreamConfig.STREAM_PREFIX() + StreamConfig.BOOTSTRAP(), systemStream.getSystem(), systemStream.getStream()), "true"); }); } } }); }
String systemStreamName = systemStreamPartition.substring(0, hashPosition); String partitionSegment = systemStreamPartition.substring(hashPosition + 1); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(systemStreamName);
StreamEdge(StreamSpec streamSpec, boolean isIntermediate, boolean isBroadcast, Config config) { this.streamSpec = streamSpec; this.isIntermediate = isIntermediate; this.isBroadcast = isBroadcast; this.config = config; if (isBroadcast) { partitions = 1; } this.name = StreamUtil.getNameFromSystemStream(getSystemStream()); }
public static Set<StreamSpec> getStreamSpecs(Set<String> streamIds, StreamConfig streamConfig) { return streamIds.stream().map(streamId -> getStreamSpec(streamId, streamConfig)).collect(Collectors.toSet()); }
JobGraph jobGraph = new JobGraph(config, appDesc); Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig); Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig); jobGraph.addSideInputStream(getStreamSpec(sideInput, streamConfig));
private void configureTables(Map<String, String> generatedConfig, Config originalConfig, Map<String, TableSpec> tables, Set<String> inputs) { generatedConfig.putAll( TableConfigGenerator.generateConfigsForTableSpecs( new MapConfig(generatedConfig), new ArrayList<>(tables.values()))); // Add side inputs to the inputs and mark the stream as bootstrap tables.values().forEach(tableSpec -> { List<String> sideInputs = tableSpec.getSideInputs(); if (sideInputs != null && !sideInputs.isEmpty()) { sideInputs.stream() .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); generatedConfig.put(String.format(StreamConfig.STREAM_PREFIX() + StreamConfig.BOOTSTRAP(), systemStream.getSystem(), systemStream.getStream()), "true"); }); } }); }
String systemStreamName = systemStreamPartition.substring(0, hashPosition); String partitionSegment = systemStreamPartition.substring(hashPosition + 1); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(systemStreamName);
StreamEdge(StreamSpec streamSpec, boolean isIntermediate, boolean isBroadcast, Config config) { this.streamSpec = streamSpec; this.isIntermediate = isIntermediate; this.isBroadcast = isBroadcast; this.config = config; if (isBroadcast) { partitions = 1; } this.name = StreamUtil.getNameFromSystemStream(getSystemStream()); }
public static Set<StreamSpec> getStreamSpecs(Set<String> streamIds, StreamConfig streamConfig) { return streamIds.stream().map(streamId -> getStreamSpec(streamId, streamConfig)).collect(Collectors.toSet()); }
JobGraph jobGraph = new JobGraph(config, appDesc); Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig); Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig); Iterable<String> sideInputs = ListUtils.emptyIfNull(localTable.getSideInputs()); for (String sideInput : sideInputs) { jobGraph.addSideInputStream(getStreamSpec(sideInput, streamConfig));
private void configureTables(Map<String, String> generatedConfig, Config originalConfig, Map<String, TableSpec> tables, Set<String> inputs) { generatedConfig.putAll( TableConfigGenerator.generateConfigsForTableSpecs( new MapConfig(generatedConfig), new ArrayList<>(tables.values()))); // Add side inputs to the inputs and mark the stream as bootstrap tables.values().forEach(tableSpec -> { List<String> sideInputs = tableSpec.getSideInputs(); if (sideInputs != null && !sideInputs.isEmpty()) { sideInputs.stream() .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); generatedConfig.put(String.format(StreamConfig.STREAM_PREFIX() + StreamConfig.BOOTSTRAP(), systemStream.getSystem(), systemStream.getStream()), "true"); }); } }); }