@Test(expected = IllegalArgumentException.class) public void testEmptyStringRejectedWithTypeInfo() { new OutputTag<>("", BasicTypeInfo.INT_TYPE_INFO); } }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
@Override public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) { if (this.outputTag == null || !this.outputTag.equals(outputTag)) { // we are only responsible for emitting to the side-output specified by our // OutputTag. return; } pushToOperator(record); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
public void setTypeSerializerSideOut(OutputTag<?> outputTag, TypeSerializer<?> serializer) { setTypeSerializer(TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), serializer); }
@Override public String toString() { return "OutputTag(" + getTypeInfo() + ", " + id + ")"; } }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
public <T> TypeSerializer<T> getTypeSerializerSideOut(OutputTag<?> outputTag, ClassLoader cl) { Preconditions.checkNotNull(outputTag, "Side output id must not be null."); try { return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), cl); } catch (Exception e) { throw new StreamTaskException("Could not instantiate serializer.", e); } }
public SideOutputTransformation(StreamTransformation<?> input, final OutputTag<T> tag) { super("SideOutput", tag.getTypeInfo(), requireNonNull(input).getParallelism()); this.input = input; this.tag = requireNonNull(tag); }
@Test(expected = NullPointerException.class) public void testNullRejectedWithTypeInfo() { new OutputTag<>(null, BasicTypeInfo.INT_TYPE_INFO); }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
@Override protected <X> void pushToOperator(StreamRecord<X> record) { try { // we know that the given outputTag matches our OutputTag so the record // must be of the type that our operator (and Serializer) expects. @SuppressWarnings("unchecked") StreamRecord<T> castRecord = (StreamRecord<T>) record; numRecordsIn.inc(); StreamRecord<T> copy = castRecord.copy(serializer.copy(castRecord.getValue())); operator.setKeyContextElement1(copy); operator.processElement(copy); } catch (ClassCastException e) { if (outputTag != null) { // Enrich error message ClassCastException replace = new ClassCastException( String.format( "%s. Failed to push OutputTag with id '%s' to operator. " + "This can occur when multiple OutputTags with different types " + "but identical names are being used.", e.getMessage(), outputTag.getId())); throw new ExceptionInChainedOperatorException(replace); } else { throw new ExceptionInChainedOperatorException(e); } } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } } }
config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) ); config.setTypeSerializerSideOut( edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()) );
@Override public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) { if (this.outputTag == null || !this.outputTag.equals(outputTag)) { // we are only responsible for emitting to the side-output specified by our // OutputTag. return; } pushToRecordWriter(record); }
@Test(expected = NullPointerException.class) public void testNullRejected() { new OutputTag<Integer>(null); }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
public <OutT> DataStream<OutT> getResultStream(OutputTag<OutT> output) { requireNonNull(output); @SuppressWarnings("unchecked") DataStream<OutT> stream = (DataStream<OutT>) resultStreams.get(output.getId()); if (stream == null) { throw new IllegalArgumentException("unknown stream named '" + output.getId() + "'."); } return stream; } }
@Override public String toString() { return "OutputTag(" + getTypeInfo() + ", " + id + ")"; } }