@Override public String toString() { return "OutputTag(" + getTypeInfo() + ", " + id + ")"; } }
public SideOutputTransformation(StreamTransformation<?> input, final OutputTag<T> tag) { super("SideOutput", tag.getTypeInfo(), requireNonNull(input).getParallelism()); this.input = input; this.tag = requireNonNull(tag); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) ); config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) );
@Override public String toString() { return "OutputTag(" + getTypeInfo() + ", " + id + ")"; } }
@Override public String toString() { return "OutputTag(" + getTypeInfo() + ", " + id + ")"; } }
public SideOutputTransformation(StreamTransformation<?> input, final OutputTag<T> tag) { super("SideOutput", tag.getTypeInfo(), requireNonNull(input).getParallelism()); this.input = input; this.tag = requireNonNull(tag); }
public SideOutputTransformation(StreamTransformation<?> input, final OutputTag<T> tag) { super("SideOutput", tag.getTypeInfo(), requireNonNull(input).getParallelism()); this.input = input; this.tag = requireNonNull(tag); }
public SideOutputTransformation(StreamTransformation<?> input, final OutputTag<T> tag) { super("SideOutput", tag.getTypeInfo(), requireNonNull(input).getParallelism()); this.input = input; this.tag = requireNonNull(tag); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) ); config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) );
config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) ); config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) );
config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) ); config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) );