/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Transforms a {@code UnionTransformation}. * * <p>This is easy, we only have to transform the inputs and return all the IDs in a list so * that downstream operations can connect to all upstream nodes. */ private <T> Collection<Integer> transformUnion(UnionTransformation<T> union) { List<StreamTransformation<T>> inputs = union.getInputs(); List<Integer> resultIds = new ArrayList<>(); for (StreamTransformation<T> input: inputs) { resultIds.addAll(transform(input)); } return resultIds; }
/** * Creates a new {@code UnionTransformation} from the given input {@code StreamTransformations}. * * <p>The input {@code StreamTransformations} must all have the same type. * * @param inputs The list of input {@code StreamTransformations} */ public UnionTransformation(List<StreamTransformation<T>> inputs) { super("Union", inputs.get(0).getOutputType(), inputs.get(0).getParallelism()); for (StreamTransformation<T> input: inputs) { if (!input.getOutputType().equals(getOutputType())) { throw new UnsupportedOperationException("Type mismatch in input " + input); } } this.inputs = Lists.newArrayList(inputs); }
private <T> void validateSplitTransformation(StreamTransformation<T> input) { if (input instanceof SelectTransformation || input instanceof SplitTransformation) { throw new IllegalStateException("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof SideOutputTransformation) { throw new IllegalStateException("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof UnionTransformation) { for (StreamTransformation<T> transformation : ((UnionTransformation<T>) input).getInputs()) { validateSplitTransformation(transformation); } } else if (input instanceof PartitionTransformation) { validateSplitTransformation(((PartitionTransformation) input).getInput()); } else { return; } } }
/** * Creates a new {@code UnionTransformation} from the given input {@code StreamTransformations}. * * <p>The input {@code StreamTransformations} must all have the same type. * * @param inputs The list of input {@code StreamTransformations} */ public UnionTransformation(List<StreamTransformation<T>> inputs) { super("Union", inputs.get(0).getOutputType(), inputs.get(0).getParallelism()); for (StreamTransformation<T> input: inputs) { if (!input.getOutputType().equals(getOutputType())) { throw new UnsupportedOperationException("Type mismatch in input " + input); } } this.inputs = Lists.newArrayList(inputs); }
/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Transforms a {@code UnionTransformation}. * * <p>This is easy, we only have to transform the inputs and return all the IDs in a list so * that downstream operations can connect to all upstream nodes. */ private <T> Collection<Integer> transformUnion(UnionTransformation<T> union) { List<StreamTransformation<T>> inputs = union.getInputs(); List<Integer> resultIds = new ArrayList<>(); for (StreamTransformation<T> input: inputs) { resultIds.addAll(transform(input)); } return resultIds; }
/** * Creates a new {@code UnionTransformation} from the given input {@code StreamTransformations}. * * <p>The input {@code StreamTransformations} must all have the same type. * * @param inputs The list of input {@code StreamTransformations} */ public UnionTransformation(List<StreamTransformation<T>> inputs) { super("Union", inputs.get(0).getOutputType(), inputs.get(0).getParallelism()); for (StreamTransformation<T> input: inputs) { if (!input.getOutputType().equals(getOutputType())) { throw new UnsupportedOperationException("Type mismatch in input " + input); } } this.inputs = Lists.newArrayList(inputs); }
/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Transforms a {@code UnionTransformation}. * * <p>This is easy, we only have to transform the inputs and return all the IDs in a list so * that downstream operations can connect to all upstream nodes. */ private <T> Collection<Integer> transformUnion(UnionTransformation<T> union) { List<StreamTransformation<T>> inputs = union.getInputs(); List<Integer> resultIds = new ArrayList<>(); for (StreamTransformation<T> input: inputs) { resultIds.addAll(transform(input)); } return resultIds; }
/** * Creates a new {@code UnionTransformation} from the given input {@code StreamTransformations}. * * <p>The input {@code StreamTransformations} must all have the same type. * * @param inputs The list of input {@code StreamTransformations} */ public UnionTransformation(List<StreamTransformation<T>> inputs) { super("Union", inputs.get(0).getOutputType(), inputs.get(0).getParallelism()); for (StreamTransformation<T> input: inputs) { if (!input.getOutputType().equals(getOutputType())) { throw new UnsupportedOperationException("Type mismatch in input " + input); } } this.inputs = Lists.newArrayList(inputs); }
/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Transforms a {@code UnionTransformation}. * * <p>This is easy, we only have to transform the inputs and return all the IDs in a list so * that downstream operations can connect to all upstream nodes. */ private <T> Collection<Integer> transformUnion(UnionTransformation<T> union) { List<StreamTransformation<T>> inputs = union.getInputs(); List<Integer> resultIds = new ArrayList<>(); for (StreamTransformation<T> input: inputs) { resultIds.addAll(transform(input)); } return resultIds; }
private <T> void validateSplitTransformation(StreamTransformation<T> input) { if (input instanceof SelectTransformation || input instanceof SplitTransformation) { throw new IllegalStateException("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof SideOutputTransformation) { throw new IllegalStateException("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof UnionTransformation) { for (StreamTransformation<T> transformation : ((UnionTransformation<T>) input).getInputs()) { validateSplitTransformation(transformation); } } else if (input instanceof PartitionTransformation) { validateSplitTransformation(((PartitionTransformation) input).getInput()); } else { return; } } }