public void setTypeSerializerSideOut(OutputTag<?> outputTag, TypeSerializer<?> serializer) { setTypeSerializer(TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), serializer); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
public <T> TypeSerializer<T> getTypeSerializerSideOut(OutputTag<?> outputTag, ClassLoader cl) { Preconditions.checkNotNull(outputTag, "Side output id must not be null."); try { return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), cl); } catch (Exception e) { throw new StreamTaskException("Could not instantiate serializer.", e); } }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
@Override protected <X> void pushToOperator(StreamRecord<X> record) { try { // we know that the given outputTag matches our OutputTag so the record // must be of the type that our operator (and Serializer) expects. @SuppressWarnings("unchecked") StreamRecord<T> castRecord = (StreamRecord<T>) record; numRecordsIn.inc(); StreamRecord<T> copy = castRecord.copy(serializer.copy(castRecord.getValue())); operator.setKeyContextElement1(copy); operator.processElement(copy); } catch (ClassCastException e) { if (outputTag != null) { // Enrich error message ClassCastException replace = new ClassCastException( String.format( "%s. Failed to push OutputTag with id '%s' to operator. " + "This can occur when multiple OutputTags with different types " + "but identical names are being used.", e.getMessage(), outputTag.getId())); throw new ExceptionInChainedOperatorException(replace); } else { throw new ExceptionInChainedOperatorException(e); } } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } } }
public <OutT> DataStream<OutT> getResultStream(OutputTag<OutT> output) { requireNonNull(output); @SuppressWarnings("unchecked") DataStream<OutT> stream = (DataStream<OutT>) resultStreams.get(output.getId()); if (stream == null) { throw new IllegalArgumentException("unknown stream named '" + output.getId() + "'."); } return stream; } }
public <OutT> DataStream<OutT> getResultStream(OutputTag<OutT> output) { requireNonNull(output); @SuppressWarnings("unchecked") DataStream<OutT> stream = (DataStream<OutT>) resultStreams.get(output.getId()); if (stream == null) { throw new IllegalArgumentException("unknown stream named '" + output.getId() + "'."); } return stream; } }
public void setTypeSerializerSideOut(OutputTag<?> outputTag, TypeSerializer<?> serializer) { setTypeSerializer(TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), serializer); }
public void setTypeSerializerSideOut(OutputTag<?> outputTag, TypeSerializer<?> serializer) { setTypeSerializer(TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), serializer); }
public void setTypeSerializerSideOut(OutputTag<?> outputTag, TypeSerializer<?> serializer) { setTypeSerializer(TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), serializer); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
/** * Adds a new virtual node that is used to connect a downstream vertex to only the outputs with * the selected side-output {@link OutputTag}. * * @param originalId ID of the node that should be connected to. * @param virtualId ID of the virtual node. * @param outputTag The selected side-output {@code OutputTag}. */ public void addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) { if (virtualSideOutputNodes.containsKey(virtualId)) { throw new IllegalStateException("Already has virtual output node with id " + virtualId); } // verify that we don't already have a virtual node for the given originalId/outputTag // combination with a different TypeInformation. This would indicate that someone is trying // to read a side output from an operation with a different type for the same side output // id. for (Tuple2<Integer, OutputTag> tag : virtualSideOutputNodes.values()) { if (!tag.f0.equals(originalId)) { // different source operator continue; } if (tag.f1.getId().equals(outputTag.getId()) && !tag.f1.getTypeInfo().equals(outputTag.getTypeInfo())) { throw new IllegalArgumentException("Trying to add a side output for the same " + "side-output id with a different type. This is not allowed. Side-output ID: " + tag.f1.getId()); } } virtualSideOutputNodes.put(virtualId, new Tuple2<>(originalId, outputTag)); }
public <T> TypeSerializer<T> getTypeSerializerSideOut(OutputTag<?> outputTag, ClassLoader cl) { Preconditions.checkNotNull(outputTag, "Side output id must not be null."); try { return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), cl); } catch (Exception e) { throw new StreamTaskException("Could not instantiate serializer.", e); } }
public <T> TypeSerializer<T> getTypeSerializerSideOut(OutputTag<?> outputTag, ClassLoader cl) { Preconditions.checkNotNull(outputTag, "Side output id must not be null."); try { return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), cl); } catch (Exception e) { throw new StreamTaskException("Could not instantiate serializer.", e); } }
public <T> TypeSerializer<T> getTypeSerializerSideOut(OutputTag<?> outputTag, ClassLoader cl) { Preconditions.checkNotNull(outputTag, "Side output id must not be null."); try { return InstantiationUtil.readObjectFromConfig(this.config, TYPE_SERIALIZER_SIDEOUT_PREFIX + outputTag.getId(), cl); } catch (Exception e) { throw new StreamTaskException("Could not instantiate serializer.", e); } }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
@Override public ResultStreams translate(String name, List<InputAndSpec<?, ?>> streamLedgerSpecs) { List<OutputTag<?>> sideOutputTags = createSideOutputTags(streamLedgerSpecs); // the input stream is a union of different streams. KeyedStream<TaggedElement, Boolean> input = union(streamLedgerSpecs) .keyBy(unused -> true); // main pipeline String serialTransactorName = "SerialTransactor(" + name + ")"; SingleOutputStreamOperator<Void> resultStream = input .process(new SerialTransactor(specs(streamLedgerSpecs), sideOutputTags)) .name(serialTransactorName) .uid(serialTransactorName + "___SERIAL_TX") .forceNonParallel() .returns(Void.class); // gather the sideOutputs. Map<String, DataStream<?>> output = new HashMap<>(); for (OutputTag<?> outputTag : sideOutputTags) { DataStream<?> rs = resultStream.getSideOutput(outputTag); output.put(outputTag.getId(), rs); } return new ResultStreams(output); } }
@Override public ResultStreams translate(String name, List<InputAndSpec<?, ?>> streamLedgerSpecs) { List<OutputTag<?>> sideOutputTags = createSideOutputTags(streamLedgerSpecs); // the input stream is a union of different streams. KeyedStream<TaggedElement, Boolean> input = union(streamLedgerSpecs) .keyBy(unused -> true); // main pipeline String serialTransactorName = "SerialTransactor(" + name + ")"; SingleOutputStreamOperator<Void> resultStream = input .process(new SerialTransactor(specs(streamLedgerSpecs), sideOutputTags)) .name(serialTransactorName) .uid(serialTransactorName + "___SERIAL_TX") .forceNonParallel() .returns(Void.class); // gather the sideOutputs. Map<String, DataStream<?>> output = new HashMap<>(); for (OutputTag<?> outputTag : sideOutputTags) { DataStream<?> rs = resultStream.getSideOutput(outputTag); output.put(outputTag.getId(), rs); } return new ResultStreams(output); } }