@Override public String toString() { return getClass().getSimpleName() + " - " + getName(); } }
/** * Takes a list of operators and creates a cascade of unions of this inputs, if needed. * If not needed (there was only one operator in the list), then that operator is returned. * * @param operators The operators. * @return The single operator or a cascade of unions of the operators. */ public static <T> Operator<T> createUnionCascade(Operator<T>... operators) { return createUnionCascade(null, operators); }
@Override public boolean preVisit(Operator<?> visitable) { if (this.seen.add(visitable)) { // add to the map final String name = visitable.getName(); List<Operator<?>> list = this.map.get(name); if (list == null) { list = new ArrayList<Operator<?>>(2); this.map.put(name, list); } list.add(visitable); // recurse into bulk iterations if (visitable instanceof BulkIteration) { ((BulkIteration) visitable).getNextPartialSolution().accept(this); } else if (visitable instanceof DeltaIteration) { ((DeltaIteration) visitable).getSolutionSetDelta().accept(this); ((DeltaIteration) visitable).getNextWorkset().accept(this); } return true; } else { return false; } }
@Override protected eu.stratosphere.api.common.operators.base.FilterOperatorBase<T, GenericFlatMap<T,T>> translateToDataFlow(Operator<T> input) { String name = getName() != null ? getName() : function.getClass().getName(); // create operator PlanFilterOperator<T> po = new PlanFilterOperator<T>(function, name, getInputType()); // set input po.setInput(input); // set dop if(this.getParallelism() > 0) { // use specified dop po.setDegreeOfParallelism(this.getParallelism()); } else { // if no dop has been specified, use dop of input operator to enable chaining po.setDegreeOfParallelism(input.getDegreeOfParallelism()); } return po; } }
if (n instanceof DataSinkNode) { type = "sink"; contents = n.getPactContract().toString(); } else if (n instanceof DataSourceNode) { type = "source"; contents = n.getPactContract().toString(); } else if (n instanceof BulkIterationNode) { type = "bulk_iteration"; contents = n.getPactContract().getName(); } else if (n instanceof WorksetIterationNode) { type = "workset_iteration"; contents = n.getPactContract().getName(); } else if (n instanceof BinaryUnionNode) { type = "pact"; } else { type = "pact"; contents = n.getPactContract().getName(); if (connNum == 0) { child1name += child1name.length() > 0 ? ", " : ""; child1name += source.getOptimizerNode().getPactContract().getName(); } else if (connNum == 1) { child2name += child2name.length() > 0 ? ", " : ""; child2name = source.getOptimizerNode().getPactContract().getName(); if (n.getPactContract().getCompilerHints() != null) { CompilerHints hints = n.getPactContract().getCompilerHints(); CompilerHints defaults = new CompilerHints();
public Union(Operator<T> input1, Operator<T> input2) { this(new BinaryOperatorInformation<T, T, T>(input1.getOperatorInfo().getOutputType(), input1.getOperatorInfo().getOutputType(), input1.getOperatorInfo().getOutputType())); setFirstInput(input1); setSecondInput(input2); } }
private JobInputVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final JobInputVertex vertex = new JobInputVertex(node.getNodeName(), this.jobGraph); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); // set task class @SuppressWarnings("unchecked") final Class<AbstractInputTask<?>> clazz = (Class<AbstractInputTask<?>>) (Class<?>) DataSourceTask.class; vertex.setInputClass(clazz); // set user code config.setStubWrapper(node.getPactContract().getUserCodeWrapper()); config.setStubParameters(node.getPactContract().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
@Override public boolean preVisit(final Operator visitable) { int degree = GenericTestPlan.this.getDegreeOfParallelism(); if (visitable instanceof GenericDataSource<?>) degree = 1; else if (degree > 1 && visitable instanceof FileDataSink) try { final Path path = new Path(((FileDataSink) visitable).getFilePath()); final FileSystem fs = path.getFileSystem(); final FileStatus f = fs.getFileStatus(path); if (!f.isDir()) { fs.delete(path, false); fs.mkdirs(path); } } catch (final IOException e) { e.printStackTrace(); } if (visitable.getDegreeOfParallelism() == -1) visitable.setDegreeOfParallelism(degree); return true; } });
protected void readUniqueFieldsAnnotation() { if (this.pactContract.getCompilerHints() != null) { Set<FieldSet> uniqueFieldSets = pactContract.getCompilerHints().getUniqueFields(); if (uniqueFieldSets != null) { if (this.uniqueFields == null) { this.uniqueFields = new HashSet<FieldSet>(); } this.uniqueFields.addAll(uniqueFieldSets); } } }
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) { // select the name (or create a default one) String name = this.name != null ? this.name : this.format.toString(); GenericDataSinkBase<T> sink = new GenericDataSinkBase<T>(this.format, new UnaryOperatorInformation<T, Nothing>(this.type, new NothingTypeInfo()), name); // set input sink.setInput(input); // set dop if(this.dop > 0) { // use specified dop sink.setDegreeOfParallelism(this.dop); } else { // if no dop has been specified, use dop of input operator to enable chaining sink.setDegreeOfParallelism(input.getDegreeOfParallelism()); } return sink; }
if (n instanceof DataSinkNode) { type = "sink"; contents = n.getPactContract().toString(); } else if (n instanceof DataSourceNode) { type = "source"; contents = n.getPactContract().toString(); } else if (n instanceof BulkIterationNode) { type = "bulk_iteration"; contents = n.getPactContract().getName(); } else if (n instanceof WorksetIterationNode) { type = "workset_iteration"; contents = n.getPactContract().getName(); } else if (n instanceof BinaryUnionNode) { type = "pact"; } else { type = "pact"; contents = n.getPactContract().getName(); if (connNum == 0) { child1name += child1name.length() > 0 ? ", " : ""; child1name += source.getOptimizerNode().getPactContract().getName(); } else if (connNum == 1) { child2name += child2name.length() > 0 ? ", " : ""; child2name = source.getOptimizerNode().getPactContract().getName(); if (n.getPactContract().getCompilerHints() != null) { CompilerHints hints = n.getPactContract().getCompilerHints(); CompilerHints defaults = new CompilerHints();
@Override public UnaryOperatorInformation<OUT, OUT> getOperatorInfo() { TypeInformation<OUT> previousOut = input.getOperatorInfo().getOutputType(); return new UnaryOperatorInformation<OUT, OUT>(previousOut, previousOut); } }
private JobInputVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final JobInputVertex vertex = new JobInputVertex(node.getNodeName(), this.jobGraph); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); // set task class @SuppressWarnings("unchecked") final Class<AbstractInputTask<?>> clazz = (Class<AbstractInputTask<?>>) (Class<?>) DataSourceTask.class; vertex.setInputClass(clazz); // set user code config.setStubWrapper(node.getPactContract().getUserCodeWrapper()); config.setStubParameters(node.getPactContract().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
protected void readUniqueFieldsAnnotation() { if (this.pactContract.getCompilerHints() != null) { Set<FieldSet> uniqueFieldSets = pactContract.getCompilerHints().getUniqueFields(); if (uniqueFieldSets != null) { if (this.uniqueFields == null) { this.uniqueFields = new HashSet<FieldSet>(); } this.uniqueFields.addAll(uniqueFieldSets); } } }
@Override public String toString() { return getClass().getSimpleName() + " - " + getName(); } }
@Override protected eu.stratosphere.api.common.operators.base.MapOperatorBase<IN, OUT, GenericMap<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : function.getClass().getName(); // create operator MapOperatorBase<IN, OUT, GenericMap<IN, OUT>> po = new MapOperatorBase<IN, OUT, GenericMap<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set dop if(this.getParallelism() > 0) { // use specified dop po.setDegreeOfParallelism(this.getParallelism()); } else { // if no dop has been specified, use dop of input operator to enable chaining po.setDegreeOfParallelism(input.getDegreeOfParallelism()); } return po; }
/** * Takes a list of operators and creates a cascade of unions of this inputs, if needed. * If not needed (there was only one operator in the list), then that operator is returned. * * @param operators The operators. * @return The single operator or a cascade of unions of the operators. */ public static <T> Operator<T> createUnionCascade(Operator<T>... operators) { return createUnionCascade(null, operators); }
public Union(Operator<T> input1, Operator<T> input2) { this(new BinaryOperatorInformation<T, T, T>(input1.getOperatorInfo().getOutputType(), input1.getOperatorInfo().getOutputType(), input1.getOperatorInfo().getOutputType())); setFirstInput(input1); setSecondInput(input2); } }