@Override public void reduce(Pair<Utf8,Long> wc, Collector<Pair<Utf8,Long>> c) { sum += wc.value(); }
@Override public void reduceFlush(Pair<Utf8,Long> wc, Collector<Pair<Utf8,Long>> c) throws IOException { wc.value(sum); c.collect(wc); sum = 0; }
public void collect(OUT datum) throws IOException { if (isMapOnly) { wrapper.datum(datum); collector.collect((KO)wrapper, (VO)NullWritable.get()); } else { // split a pair Pair<K,V> pair = (Pair<K,V>)datum; keyWrapper.datum(pair.key()); valueWrapper.datum(pair.value()); collector.collect((KO)keyWrapper, (VO)valueWrapper); } } }
@Override public Pair<K,V> next(Pair<K,V> reuse) throws IOException { prepare(); if (!hasNext()) throw new NoSuchElementException(); Pair<K,V> result = reuse; if (result == null) result = new Pair<>(schema); result.key(keyConverter.convert(key)); reader.getCurrentValue(value); result.value(valConverter.convert(value)); // swap key and spareKey Writable k = key; key = spareKey; spareKey = k; ready = false; return result; }
public void checkFile(FileReader<Pair<Long,CharSequence>> reader) throws Exception { long i = 0; for (Pair<Long,CharSequence> p : reader) { assertEquals((Long)i, p.key()); assertEquals(Long.toString(i), p.value().toString()); i++; } assertEquals(COUNT, i); reader.close(); }
public static void validateCountsFile(File file) throws Exception { int numWords = 0; DatumReader<Pair<Utf8,Long>> reader = new SpecificDatumReader<>(); try(InputStream in = new BufferedInputStream(new FileInputStream(file))) { try (DataFileStream<Pair<Utf8, Long>> counts = new DataFileStream<>(in, reader)) { for (Pair<Utf8, Long> wc : counts) { assertEquals(wc.key().toString(), COUNTS.get(wc.key().toString()), wc.value()); numWords++; } checkMeta(counts); } } assertEquals(COUNTS.size(), numWords); }
DataFileStream<Pair<Utf8, Long>> counts = new DataFileStream<>(cin, reader); for (Pair<Utf8, Long> wc : counts) { assertEquals(wc.key().toString(), WordCountUtil.COUNTS.get(wc.key().toString()), wc.value()); numWords++;
int numWords = 0; for (Pair<Utf8, Long> wc : counts) { assertEquals(wc.key().toString(), WordCountUtil.COUNTS.get(wc.key().toString()), wc.value()); numWords++;
public void validateCountsFile() throws Exception { AvroColumnReader<Pair<String,Long>> reader = new AvroColumnReader<>( new AvroColumnReader.Params(countFiles).setModel(SpecificData.get())); int numWords = 0; for (Pair<String,Long> wc : reader) { assertEquals(wc.key(), COUNTS.get(wc.key()), wc.value()); numWords++; } reader.close(); assertEquals(COUNTS.size(), numWords); }
@Override public void consume(Object object) { Pair record = (Pair) object; _result.put(record.key().toString(), Double.parseDouble(record.value().toString())); } @Override
@Override public org.apache.avro.mapred.Pair map(Pair input) { if (pairSchema == null) { pairSchema = new Schema.Parser().parse(pairSchemaJson); } org.apache.avro.mapred.Pair avroPair = new org.apache.avro.mapred.Pair( pairSchema); avroPair.key(keyMapFn.map(input.first())); avroPair.value(valueMapFn.map(input.second())); return avroPair; } }
@Override public org.apache.avro.mapred.Pair map(Pair input) { if (pairSchema == null) { pairSchema = new Schema.Parser().parse(pairSchemaJson); } org.apache.avro.mapred.Pair avroPair = new org.apache.avro.mapred.Pair(pairSchema); avroPair.key(keyMapFn.map(input.first())); avroPair.value(valueMapFn.map(input.second())); return avroPair; } }
@Override public void map(Pair<CharSequence, LabelSenseList> pair, AvroCollector<Pair<Integer, PrimaryLabels>> collector, Reporter reporter) throws IOException { CharSequence label = pair.key() ; LabelSenseList senses = pair.value() ; if (senses.getSenses().isEmpty()) return ; LabelSense firstSense = senses.getSenses().get(0) ; ArrayList<CharSequence> primaryLabels = new ArrayList<CharSequence>() ; primaryLabels.add(label) ; collector.collect(new Pair<Integer, PrimaryLabels>(firstSense.getId(), new PrimaryLabels(primaryLabels))); } }
@Override public org.apache.avro.mapred.Pair map(Pair input) { if (pairSchema == null) { pairSchema = new Schema.Parser().parse(pairSchemaJson); } org.apache.avro.mapred.Pair avroPair = new org.apache.avro.mapred.Pair(pairSchema); avroPair.key(keyMapFn.map(input.first())); avroPair.value(valueMapFn.map(input.second())); return avroPair; } }
public void collect(OUT datum) throws IOException { if (isMapOnly) { wrapper.datum(datum); collector.collect((KO)wrapper, (VO)NullWritable.get()); } else { // split a pair Pair<K,V> pair = (Pair<K,V>)datum; keyWrapper.datum(pair.key()); valueWrapper.datum(pair.value()); collector.collect((KO)keyWrapper, (VO)valueWrapper); } } }
throw new IOException("Mapper not configured with root category title") ; PageDetail page = pair.value() ;
@Override public Pair<K,V> next(Pair<K,V> reuse) throws IOException { prepare(); if (!hasNext()) throw new NoSuchElementException(); Pair<K,V> result = reuse; if (result == null) result = new Pair<K,V>(schema); result.key(keyConverter.convert(key)); reader.getCurrentValue(value); result.value(valConverter.convert(value)); // swap key and spareKey Writable k = key; key = spareKey; spareKey = k; ready = false; return result; }
@Override public void map(Pair<PageKey, PageDetail> pair, AvroCollector<Pair<Integer, PageDetail>> collector, Reporter reporter) throws IOException { PageKey key = pair.key() ; PageDetail page = pair.value() ; page.setNamespace(key.getNamespace()); page.setTitle(key.getTitle()); collector.collect(new Pair<Integer, PageDetail>(page.getId(), page)); } }
@Override public void map(Pair<Integer, PageDepthSummary> pair, AvroCollector<Pair<Integer, PageDepthSummary>> collector, Reporter reporter) throws IOException { Integer id = pair.key() ; PageDepthSummary depthSummary = pair.value() ; if (depthSummary.getDepthForwarded()) { //if we have already processed this in previous iterations, just pass it along directly InitialDepthMapper.collect(id, depthSummary, collector); return ; } if (depthSummary.getDepth() == null) { //if we haven't reached this node yet, just pass it along directly InitialDepthMapper.collect(id, depthSummary, collector); return ; } InitialDepthMapper.shareDepth(depthSummary, collector, reporter) ; InitialDepthMapper.collect(id, depthSummary, collector); }
public static void validateCountsFile() throws Exception { AvroColumnReader<Pair<String,Long>> reader = new AvroColumnReader<Pair<String,Long>> (new AvroColumnReader.Params(COUNTS_FILE).setModel(SpecificData.get())); int numWords = 0; for (Pair<String,Long> wc : reader) { assertEquals(wc.key(), COUNTS.get(wc.key()), wc.value()); numWords++; } reader.close(); assertEquals(COUNTS.size(), numWords); }