/** * Creates a new instance. * * @param typeClass type of the stream elements */ public CountOperator(Class<Type> typeClass) { this(DataSetType.createDefault(typeClass)); }
/** * Find out whether this corresponds to either {@link #none()} or {@link #groupedNone()}. * * @return whether above condition holds */ public boolean isNone() { return this.equals(none()) || this.equals(groupedNone()); }
public CartesianOperator(Class<InputType0> inputType0Class, Class<InputType1> inputType1Class) { super(DataSetType.createDefault(inputType0Class), DataSetType.createDefault(inputType1Class), DataSetType.createDefaultUnchecked(Tuple2.class), true); }
/** * Creates a new instance. * * @param typeClass the class of data quanta being grouped */ public GlobalMaterializedGroupOperator(Class<Type> typeClass) { this(DataSetType.createDefault(typeClass), DataSetType.createGrouped(typeClass)); }
private SubplanPattern createSubplanPattern() { final OperatorPattern operatorPattern = new OperatorPattern( "group", new GlobalMaterializedGroupOperator<>(DataSetType.none(), DataSetType.groupedNone()), false ); return SubplanPattern.createSingleton(operatorPattern); }
private SubplanPattern createSubplanPattern() { final OperatorPattern operatorPattern = new OperatorPattern<>( "sink", new TextFileSink<>(null, DataSetType.none().getDataUnitType().getTypeClass()), false ); return SubplanPattern.createSingleton(operatorPattern); }
public JavaTsvFileSink(String targetPath, DataSetType<T> type) { super(type); assert type.equals(DataSetType.createDefault(Tuple2.class)) : String.format("Illegal type for %s: %s", this, type); this.targetPath = targetPath; }
/** * Creates a new instance. * * @param type type of the dataunit elements */ public FilterOperator(DataSetType<Type> type, PredicateDescriptor.SerializablePredicate<Type> predicateDescriptor) { this(new PredicateDescriptor<>(predicateDescriptor, type.getDataUnitType().getTypeClass()), type); }
repeatOperator.getType().unchecked(), DataSetType.createDefault(Integer.class), ints -> RheemCollections.getSingle(ints) >= numIterations, numIterations
/** * Creates a new instance. */ public ZipWithIdOperator(DataSetType<InputType> inputType) { super(inputType, DataSetType.createDefaultUnchecked(Tuple2.class), false); }
public boolean isCompatibleWith(Slot<?> that) { return this.type.equals(that.type); }
private SubplanPattern createSubplanPattern() { final OperatorPattern operatorPattern = new OperatorPattern<>( "operator", new MaterializedGroupByOperator<>(null, DataSetType.none(), DataSetType.groupedNone()), false); return SubplanPattern.createSingleton(operatorPattern); }
private SubplanPattern createSubplanPattern() { final OperatorPattern operatorPattern = new OperatorPattern<>( "sink", new TextFileSink<>("", DataSetType.none().getDataUnitType().getTypeClass()), false ); return SubplanPattern.createSingleton(operatorPattern); }
public SparkTsvFileSink(String targetPath, DataSetType<T> type) { super(type); assert type.equals(DataSetType.createDefault(Tuple2.class)) : String.format("Illegal type for %s: %s", this, type); this.targetPath = targetPath; }
/** * Creates a new instance. * * @param keyDescriptor describes the key w.r.t. to the processed data units */ public GroupByOperator(TransformationDescriptor<Input, Key> keyDescriptor) { this(keyDescriptor, DataSetType.createDefault(keyDescriptor.getInputType()), DataSetType.createGrouped(keyDescriptor.getInputType())); }
private T lineParse(String line) { // TODO rewrite in less verbose way. Class typeClass = this.getType().getDataUnitType().getTypeClass(); int tabPos = line.indexOf('\t'); if (tabPos == -1) { if (typeClass == Integer.class) { return (T) Integer.valueOf(line); } else if (typeClass == Float.class) { return (T) Float.valueOf(line); } else if (typeClass == String.class) { return (T) String.valueOf(line); } else throw new RheemException(String.format("Cannot parse TSV file line %s", line)); } else if (typeClass == Record.class) { // TODO: Fix Record parsing. return (T) new Record(); } else if (typeClass == Tuple2.class) { // TODO: Fix Tuple2 parsing return (T) new Tuple2( Integer.valueOf(line.substring(0, tabPos)), Float.valueOf(line.substring(tabPos + 1))); } else throw new RheemException(String.format("Cannot parse TSV file line %s", line)); }
private static <InputType0, InputType1> DataSetType<Tuple2<InputType0, InputType1>> createOutputDataSetType() { return DataSetType.createDefaultUnchecked(Tuple2.class); }
/** * Compares the inputs of two operators and passes quietly if they are identical. * * @throws IllegalArgumentException if the operators differ in their inputs */ public static void assertEqualInputs(Operator o1, Operator o2) throws IllegalArgumentException { if (o1.getNumInputs() != o2.getNumInputs()) { throw new IllegalArgumentException(String.format("%s and %s have different numbers of inputs.", o1, o2)); } for (int i = 0; i < o1.getNumInputs(); i++) { final InputSlot<?> input1 = o1.getInput(i); final InputSlot<?> input2 = o2.getInput(i); if ((input1 == null && input2 != null) || (input1 != null && input2 == null) || (input1 != null && input2 != null && !input1.getType().equals(input2.getType()))) { throw new IllegalArgumentException("Operators differ in input " + i + "."); } } }