@Override public void configure(Configuration conf) { intern.configure(conf); }
@Override public void configure(Configuration conf) { intern.configure(conf); }
@Override public void configure(Configuration conf) { delegate.configure(conf); fn.configure(conf); }
public RTNode toRTNode(boolean inputNode, Configuration conf, NodeContext nodeContext) { List<RTNode> childRTNodes = Lists.newArrayList(); fn.configure(conf); for (DoNode child : children) { childRTNodes.add(child.toRTNode(false, conf, nodeContext)); } Converter inputConverter = null; if (inputNode) { if (nodeContext == NodeContext.MAP) { inputConverter = ptype.getConverter(); } else { inputConverter = ((PGroupedTableType<?,?>) ptype).getGroupingConverter(); } } return new RTNode(fn, name, childRTNodes, inputConverter, outputConverter, outputName); }
public RTNode toRTNode(boolean inputNode, Configuration conf, NodeContext nodeContext) { List<RTNode> childRTNodes = Lists.newArrayList(); fn.configure(conf); for (DoNode child : children) { childRTNodes.add(child.toRTNode(false, conf, nodeContext)); } Converter inputConverter = null; if (inputNode) { if (nodeContext == NodeContext.MAP) { inputConverter = ptype.getConverter(); } else { inputConverter = ((PGroupedTableType<?, ?>) ptype).getGroupingConverter(); } } return new RTNode(fn, (PType<Object>) getPType(), name, childRTNodes, inputConverter, outputConverter, outputName); }
public RTNode toRTNode(boolean inputNode, Configuration conf, NodeContext nodeContext) { List<RTNode> childRTNodes = Lists.newArrayList(); if (options != null) { options.configure(conf); } fn.configure(conf); for (DoNode child : children) { childRTNodes.add(child.toRTNode(false, conf, nodeContext)); } Converter inputConverter = null; if (inputNode) { if (nodeContext == NodeContext.MAP) { inputConverter = source.getConverter(); } else { inputConverter = ((PGroupedTableType<?, ?>) ptype).getGroupingConverter(); } } return new RTNode(fn, (PType<Object>) getPType(), name, childRTNodes, inputConverter, outputConverter, outputName); }
private JavaRDDLike<?, ?> getJavaRDDLikeInternal(SparkRuntime runtime) { JavaRDDLike<?, ?> parentRDD = ((SparkCollection) getOnlyParent()).getJavaRDDLike(runtime); fn.configure(runtime.getConfiguration()); return parentRDD.mapPartitionsWithIndex( new FlatMapIndexFn(fn, parentRDD instanceof JavaPairRDD, runtime.getRuntimeContext()), false); } }
private JavaRDDLike<?, ?> getJavaRDDLikeInternal(SparkRuntime runtime) { JavaRDDLike<?, ?> parentRDD = ((SparkCollection) getOnlyParent()).getJavaRDDLike(runtime); fn.configure(runtime.getConfiguration()); return parentRDD.mapPartitionsWithIndex( new FlatMapIndexFn(fn, parentRDD instanceof JavaPairRDD, runtime.getRuntimeContext()), false); } }
private JavaRDDLike<?, ?> getJavaRDDLikeInternal(SparkRuntime runtime) { if (combineFn instanceof CombineFn && getOnlyParent() instanceof PGroupedTableImpl) { runtime.setCombineFn((CombineFn) combineFn); } JavaRDDLike<?, ?> parentRDD = ((SparkCollection) getOnlyParent()).getJavaRDDLike(runtime); fn.configure(runtime.getConfiguration()); return parentRDD .mapPartitionsWithIndex( new FlatMapIndexFn(fn, parentRDD instanceof JavaPairRDD, runtime.getRuntimeContext()), false) .mapPartitionsToPair(new CrunchPairTuple2()); } }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type, ParallelDoOptions options) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); doFn.setContext(getInMemoryContext(conf)); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
private JavaRDDLike<?, ?> getJavaRDDLikeInternal(SparkRuntime runtime) { if (combineFn instanceof CombineFn && getOnlyParent() instanceof PGroupedTableImpl) { runtime.setCombineFn((CombineFn) combineFn); } JavaRDDLike<?, ?> parentRDD = ((SparkCollection) getOnlyParent()).getJavaRDDLike(runtime); fn.configure(runtime.getConfiguration()); return parentRDD .mapPartitionsWithIndex( new FlatMapIndexFn(fn, parentRDD instanceof JavaPairRDD, runtime.getRuntimeContext()), false) .mapPartitionsToPair(new CrunchPairTuple2()); } }
@Override public <T> PCollection<T> parallelDo(String name, DoFn<S, T> doFn, PType<T> type, ParallelDoOptions options) { doFn = verifySerializable(name, doFn); InMemoryEmitter<T> emitter = new InMemoryEmitter<T>(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); doFn.setContext(getInMemoryContext(conf)); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemCollection<T>(emitter.getOutput(), type, name); }