Refine search
@Override protected org.apache.flink.api.common.operators.SingleInputOperator<T, T, ?> translateToDataFlow(Operator<T> input) { // All the translation magic happens when the iteration end is encountered. throw new InvalidProgramException("A data set that is part of an iteration was used as a sink or action." + " Did you forget to close the iteration?"); } }
/** * Non grouped aggregation. */ public ScalaAggregateOperator(org.apache.flink.api.java.DataSet<IN> input, Aggregations function, int field) { super(Preconditions.checkNotNull(input), input.getType()); Preconditions.checkNotNull(function); if (!input.getType().isTupleType()) { throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types."); } TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType(); if (field < 0 || field >= inType.getArity()) { throw new IllegalArgumentException("Aggregation field position is out of range."); } AggregationFunctionFactory factory = function.getFactory(); AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass()); // this is the first aggregation operator after a regular data set (non grouped aggregation) this.aggregationFunctions.add(aggFunct); this.fields.add(field); this.grouping = null; }
/** * The purpose of this method is solely to check whether the data type to be processed * is in fact a tuple type. */ @Override public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) { if (!type.isTupleType()) { throw new InvalidProgramException("The " + CsvOutputFormat.class.getSimpleName() + " can only be used to write tuple data sets."); } } }
/** * @deprecated Deprecated method only kept for compatibility. */ @SuppressWarnings("unchecked") @Deprecated @PublicEvolving public <R extends Tuple> ProjectOperator<IN, R> types(Class<?>... types) { TupleTypeInfo<R> typeInfo = (TupleTypeInfo<R>) this.getResultType(); if (types.length != typeInfo.getArity()) { throw new InvalidProgramException("Provided types do not match projection."); } for (int i = 0; i < types.length; i++) { Class<?> typeClass = types[i]; if (!typeClass.equals(typeInfo.getTypeAt(i).getTypeClass())) { throw new InvalidProgramException("Provided type " + typeClass.getSimpleName() + " at position " + i + " does not match projection"); } } return (ProjectOperator<IN, R>) this; }
@Test public void testDefaultJoinOnTwoCustomTypeInputsWithInnerClassKeyExtractorsDisabledClosureCleaner() throws Exception { /* * (Default) Join on two custom type inputs with key extractors, check if disabling closure cleaning works */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableClosureCleaner(); DataSet<CustomType> ds1 = CollectionDataSets.getCustomTypeDataSet(env); DataSet<CustomType> ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env); boolean correctExceptionTriggered = false; try { DataSet<Tuple2<CustomType, CustomType>> joinDs = ds1.join(ds2) .where(new KeySelector<CustomType, Integer>() { @Override public Integer getKey(CustomType value) { return value.myInt; } }) .equalTo(new KeySelector<CustomType, Integer>() { @Override public Integer getKey(CustomType value) throws Exception { return value.myInt; } }); } catch (InvalidProgramException ex) { correctExceptionTriggered = (ex.getCause() instanceof java.io.NotSerializableException); } Assert.assertTrue(correctExceptionTriggered); }
throw new InvalidProgramException("Grouping is only possible on composite types."); FunctionUtils.openFunction(function, this.parameters); TypeSerializer<T> serializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
/** * Creates a new local environment that configures its local executor with the given configuration. * * @param config The configuration used to configure the local executor. */ public LocalEnvironment(Configuration config) { if (!ExecutionEnvironment.areExplicitEnvironmentsAllowed()) { throw new InvalidProgramException( "The LocalEnvironment cannot be instantiated when running in a pre-defined context " + "(such as Command Line Client, Scala Shell, or TestEnvironment)"); } this.configuration = config == null ? new Configuration() : config; }
@Override public ExecutionEnvironment createExecutionEnvironment() { if (isDetached && lastEnvCreated != null) { throw new InvalidProgramException("Multiple environments cannot be created in detached mode"); } lastEnvCreated = isDetached ? new DetachedEnvironment(client, jarFilesToAttach, classpathsToAttach, userCodeClassLoader, savepointSettings) : new ContextEnvironment(client, jarFilesToAttach, classpathsToAttach, userCodeClassLoader, savepointSettings); if (defaultParallelism > 0) { lastEnvCreated.setParallelism(defaultParallelism); } return lastEnvCreated; }
if (!type.isTupleType() || !(type instanceof CompositeType)) { throw new InvalidProgramException("Specifying keys via field positions is only valid " + "for tuple data types. Type: " + type); if (type.getArity() == 0) { throw new InvalidProgramException("Tuple size must be greater than 0. Size: " + type.getArity()); if(!ffd.getType().isKeyType()) { throw new InvalidProgramException("This type (" + ffd.getType() + ") cannot be used as key.");
@Test(expected = InvalidProgramException.class) public void testRangePartitionInIteration() throws Exception { // does not apply for collection execution if (super.mode == TestExecutionMode.COLLECTION) { throw new InvalidProgramException("Does not apply for collection execution"); } final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSource<Long> source = env.generateSequence(0, 10000); DataSet<Tuple2<Long, String>> tuples = source.map(new MapFunction<Long, Tuple2<Long, String>>() { @Override public Tuple2<Long, String> map(Long v) throws Exception { return new Tuple2<>(v, Long.toString(v)); } }); DeltaIteration<Tuple2<Long, String>, Tuple2<Long, String>> it = tuples.iterateDelta(tuples, 10, 0); DataSet<Tuple2<Long, String>> body = it.getWorkset() .partitionByRange(1) // Verify that range partition is not allowed in iteration .join(it.getSolutionSet()) .where(0).equalTo(0).projectFirst(0).projectSecond(1); DataSet<Tuple2<Long, String>> result = it.closeWith(body, body); result.collect(); // should fail }
public static boolean isSortKey(int fieldPos, TypeInformation<?> type) { if (!type.isTupleType() || !(type instanceof CompositeType)) { throw new InvalidProgramException("Specifying keys via field positions is only valid " + "for tuple data types. Type: " + type); } if (type.getArity() == 0) { throw new InvalidProgramException("Tuple size must be greater than 0. Size: " + type.getArity()); } if(fieldPos < 0 || fieldPos >= type.getArity()) { throw new IndexOutOfBoundsException("Tuple position is out of range: " + fieldPos); } TypeInformation<?> sortKeyType = ((CompositeType<?>)type).getTypeAt(fieldPos); return sortKeyType.isSortKeyType(); }
@Override public <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo) { if (keyFields.size() != 1) { throw new InvalidProgramException("Custom partitioners can only be used with keys that have one key field."); } if (typeInfo == null) { // try to extract key type from partitioner try { typeInfo = TypeExtractor.getPartitionerTypes(partitioner); } catch (Throwable t) { // best effort check, so we ignore exceptions } } // only check if type is known and not a generic type if (typeInfo != null && !(typeInfo instanceof GenericTypeInfo)) { // check equality of key and partitioner type if (!keyType.equals(typeInfo)) { throw new InvalidProgramException("The partitioner is incompatible with the key type. " + "Partitioner type: " + typeInfo + " , key type: " + keyType); } } }
/** * Generic method to create an input {@link DataSet} with in {@link InputFormat}. The DataSet will not be * immediately created - instead, this method returns a DataSet that will be lazily created from * the input format once the program is executed. * * <p>Since all data sets need specific information about their types, this method needs to determine * the type of the data produced by the input format. It will attempt to determine the data type * by reflection, unless the input format implements the {@link ResultTypeQueryable} interface. * In the latter case, this method will invoke the {@link ResultTypeQueryable#getProducedType()} * method to determine data type produced by the input format. * * @param inputFormat The input format used to create the data set. * @return A {@link DataSet} that represents the data created by the input format. * * @see #createInput(InputFormat, TypeInformation) */ public <X> DataSource<X> createInput(InputFormat<X, ?> inputFormat) { if (inputFormat == null) { throw new IllegalArgumentException("InputFormat must not be null."); } try { return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat)); } catch (Exception e) { throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " + "Please specify the TypeInformation of the produced type explicitly by using the " + "'createInput(InputFormat, TypeInformation)' method instead.", e); } }
private <K> void ensureSortableKey(Keys.SelectorFunctionKeys<T, K> sortKey) { if (!sortKey.getKeyType().isSortKeyType()) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } }
for (int i = 0; i < typeInfo.getArity(); i++) { stack.push(((TupleTypeInfoBase) typeInfo).getTypeAt(i)); throw new InvalidProgramException("Type " + keyType + " cannot be used as key. Contained " + "UNSUPPORTED key types: " + StringUtils.join(unsupportedTypes, ", ") + ". Look " + "at the keyBy() documentation for the conditions a type has to satisfy in order to be " +
public SelectorFunctionKeys(KeySelector<T, K> keyExtractor, TypeInformation<T> inputType, TypeInformation<K> keyType) { if (keyExtractor == null) { throw new NullPointerException("Key extractor must not be null."); } if (keyType == null) { throw new NullPointerException("Key type must not be null."); } if (!keyType.isKeyType()) { throw new InvalidProgramException("Return type "+keyType+" of KeySelector "+keyExtractor.getClass()+" is not a valid key type"); } this.keyExtractor = keyExtractor; this.inputType = inputType; this.keyType = keyType; this.originalKeyTypes = new TypeInformation[] {keyType}; if (keyType instanceof CompositeType) { this.keyFields = ((CompositeType<T>)keyType).getFlatFields(ExpressionKeys.SELECT_ALL_CHAR); } else { this.keyFields = new ArrayList<>(1); this.keyFields.add(new FlatFieldDescriptor(0, keyType)); } }
/** * Non grouped aggregation. */ public AggregateOperator(DataSet<IN> input, Aggregations function, int field, String aggregateLocationName) { super(Preconditions.checkNotNull(input), input.getType()); Preconditions.checkNotNull(function); this.aggregateLocationName = aggregateLocationName; if (!input.getType().isTupleType()) { throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types."); } TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType(); if (field < 0 || field >= inType.getArity()) { throw new IllegalArgumentException("Aggregation field position is out of range."); } AggregationFunctionFactory factory = function.getFactory(); AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass()); // this is the first aggregation operator after a regular data set (non grouped aggregation) this.aggregationFunctions.add(aggFunct); this.fields.add(field); this.grouping = null; }
/** * The purpose of this method is solely to check whether the data type to be processed * is in fact a tuple type. */ @Override public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) { if (!type.isTupleType()) { throw new InvalidProgramException("The " + ScalaCsvOutputFormat.class.getSimpleName() + " can only be used to write tuple data sets."); } } }
/** * @deprecated Deprecated method only kept for compatibility. */ @SuppressWarnings({ "hiding", "unchecked" }) @Deprecated @PublicEvolving public <OUT extends Tuple> CrossOperator<I1, I2, OUT> types(Class<?>... types) { TupleTypeInfo<OUT> typeInfo = (TupleTypeInfo<OUT>) this.getResultType(); if (types.length != typeInfo.getArity()) { throw new InvalidProgramException("Provided types do not match projection."); } for (int i = 0; i < types.length; i++) { Class<?> typeClass = types[i]; if (!typeClass.equals(typeInfo.getTypeAt(i).getTypeClass())) { throw new InvalidProgramException("Provided type " + typeClass.getSimpleName() + " at position " + i + " does not match projection"); } } return (CrossOperator<I1, I2, OUT>) this; }
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableClosureCleaner(); correctExceptionTriggered = (ex.getCause() instanceof java.io.NotSerializableException);