@Override public void initialize() { intern.setContext(getContext()); intern.initialize(); }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type, ParallelDoOptions options) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); doFn.setContext(getInMemoryContext(conf)); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
/** * Called during setup to pass the {@link TaskInputOutputContext} to * this {@code DoFn} instance. */ public void setContext(TaskInputOutputContext<?, ?, ?, ?> context) { this.context = context; initialize(); }
public void initialize(CrunchTaskContext ctxt) { if (emitter != null) { // Already initialized return; } fn.setContext(ctxt.getContext()); fn.initialize(); for (RTNode child : children) { child.initialize(ctxt); } if (outputConverter != null) { if (outputName != null) { this.emitter = new MultipleOutputEmitter(outputConverter, ctxt.getMultipleOutputs(), outputName); } else { this.emitter = new OutputEmitter(outputConverter, ctxt.getContext()); } } else if (!children.isEmpty()) { Configuration conf = ctxt.getContext().getConfiguration(); boolean disableDeepCopy = conf.getBoolean(RuntimeParameters.DISABLE_DEEP_COPY, false); this.emitter = new IntermediateEmitter(outputPType, children, conf, disableDeepCopy || fn.disableDeepCopy()); } else { throw new CrunchRuntimeException("Invalid RTNode config: no emitter for: " + nodeName); } }
public void process(Object input) { try { fn.process(input, emitter); } catch (CrunchRuntimeException e) { if (!e.wasLogged()) { LOG.info(String.format("Crunch exception in '%s' for input: %s", nodeName, input.toString()), e); e.markLogged(); } throw e; } }
@Override public void setContext(TaskInputOutputContext<?, ?, ?, ?> context) { super.setContext(context); postProcessFn.setContext(context); }
@Override public void incrCounter(String group, String name, long by) { super.increment(group, name, by); }
@Override protected long getSizeInternal() { long parentSize = parent.getSize(); if (parentSize == 0L) { return parentSize; } return Math.max(1L, (long) (fn.scaleFactor() * parentSize)); }
public Counters.Counter getCounter(String group, String name) { return proxyCounter(super.getCounter(group, name)); }
@Override public void cleanup(Emitter<T> emitter) { intern.cleanup(emitter); } }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type, ParallelDoOptions options) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); doFn.setContext(getInMemoryContext(getPipeline().getConfiguration())); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public void initialize() { super.initialize(); postProcessFn.initialize(); leftValueType.initialize(getConfiguration()); }
public void process(Object input) { try { fn.process(input, emitter); } catch (CrunchRuntimeException e) { if (!e.wasLogged()) { LOG.info(String.format("Crunch exception in '%s' for input: %s", nodeName, input.toString()), e); e.markLogged(); } throw e; } }
@Override public void setContext(TaskInputOutputContext<?, ?, ?, ?> context) { super.setContext(context); reduceFn.setContext(context); }
protected void increment(String groupName, String counterName) { increment(groupName, counterName, 1); }
@Override protected long getSizeInternal() { long parentSize = parent.getSize(); if (parentSize == 0L) { return parentSize; } return Math.max(1L, (long) (fn.scaleFactor() * parentSize)); }
public Counters.Counter getCounter(Enum<?> counter) { return proxyCounter(super.getCounter(counter)); }