public void process(Object input) { try { fn.process(input, emitter); } catch (CrunchRuntimeException e) { if (!e.wasLogged()) { LOG.info(String.format("Crunch exception in '%s' for input: %s", nodeName, input.toString()), e); e.markLogged(); } throw e; } }
public void process(Object input) { try { fn.process(input, emitter); } catch (CrunchRuntimeException e) { if (!e.wasLogged()) { LOG.info(String.format("Crunch exception in '%s' for input: %s", nodeName, input.toString()), e); e.markLogged(); } throw e; } }
public void process(Object input) { try { fn.process(input, emitter); } catch (CrunchRuntimeException e) { if (!e.wasLogged()) { LOG.info("Crunch exception in '{}' for input: {}", new Object[]{nodeName, input, e}); e.markLogged(); } throw e; } }
@Override public boolean hasNext() { while (cache.isEmpty() && iter.hasNext()) { fn.process(iter.next(), cache); } if (cache.isEmpty() && !cleanup) { fn.cleanup(cache); cleanup = true; } return !cache.isEmpty(); }
@Override public void process(Pair<Pair<K, V1>, Iterable<Pair<V1, V2>>> input, Emitter<T> emitter) { intern.process(Pair.of(input.first().first(), input.second()), emitter); }
@Override public void process(Pair<K, Iterable<V>> input, Emitter<T> emitter) { reduceFn.process(Pair.of(input.first(), detachIterable(input.second(), valueType)), emitter); }
@Override public void process(Pair<Pair<K, V1>, Iterable<Pair<V1, V2>>> input, Emitter<T> emitter) { intern.process(Pair.of(input.first().first(), input.second()), emitter); }
@Override public void process(Pair<K, Iterable<V>> input, Emitter<T> emitter) { reduceFn.process(Pair.of(input.first(), detachIterable(input.second(), valueType)), emitter); }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public <T> PCollection<T> parallelDo(String name, DoFn<S, T> doFn, PType<T> type) { InMemoryEmitter<T> emitter = new InMemoryEmitter<T>(); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemCollection<T>(emitter.getOutput(), type, name); }
@Override public void process(Pair<Pair<K, Integer>, Iterable<Pair<U, V>>> input, Emitter<T> emitter) { Pair<K, Integer> keyPair = input.first(); if (keyPair.second() == 0) { leftValue = leftValueType.getDetachedValue(input.second().iterator().next().first()); currentKey = input.first().first(); } else if (keyPair.second() == 1 && input.first().first().equals(currentKey)) { postProcessFn.process(Pair.of(leftValue, wrapIterable(input.second())), emitter); leftValue = null; } }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type, ParallelDoOptions options) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); doFn.setContext(getInMemoryContext(getPipeline().getConfiguration())); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public <T> PCollection<T> parallelDo(String name, DoFn<S, T> doFn, PType<T> type, ParallelDoOptions options) { InMemoryEmitter<T> emitter = new InMemoryEmitter<T>(); doFn.setContext(getInMemoryContext(getPipeline().getConfiguration())); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemCollection<T>(emitter.getOutput(), type, name); }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type, ParallelDoOptions options) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); doFn.setContext(getInMemoryContext(conf)); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public <T> PCollection<T> parallelDo(String name, DoFn<S, T> doFn, PType<T> type, ParallelDoOptions options) { doFn = verifySerializable(name, doFn); InMemoryEmitter<T> emitter = new InMemoryEmitter<T>(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); doFn.setContext(getInMemoryContext(conf)); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemCollection<T>(emitter.getOutput(), type, name); }