protected void reduce( ImmutableBytesWritable row, java.lang.Iterable<KeyValue> kvs, org.apache.hadoop.mapreduce.Reducer<ImmutableBytesWritable, KeyValue, ImmutableBytesWritable, KeyValue>.Context context) throws java.io.IOException, InterruptedException { TreeSet<KeyValue> map = new TreeSet<>(KeyValue.COMPARATOR); TreeSet<byte[]> qualifierSet = new TreeSet<>(Bytes.BYTES_COMPARATOR); for (KeyValue kv : kvs) { byte[] qualifier = CellUtil.cloneQualifier(kv); if (qualifierSet.contains(qualifier)) { throw new IllegalStateException("there is duplicate key:" + row); } qualifierSet.add(qualifier); try { map.add(kv.clone()); } catch (CloneNotSupportedException e) { throw new java.io.IOException(e); } } context.setStatus("Read " + map.getClass()); int index = 0; for (KeyValue kv : map) { context.write(row, kv); if (++index % 100 == 0) context.setStatus("Wrote " + index); } } }
if (cnt % 10 == 0) context.setStatus("Combine " + cnt); if (curSize > threshold) { if (LOG.isDebugEnabled()) {
protected void reduce(ImmutableBytesWritable row, Iterable<Cell> kvs, Reducer<ImmutableBytesWritable, Cell, ImmutableBytesWritable, Cell>.Context context) throws java.io.IOException, InterruptedException { TreeSet<Cell> map = new TreeSet<>(CellComparator.getInstance()); for (Cell kv : kvs) { try { map.add(PrivateCellUtil.deepClone(kv)); } catch (CloneNotSupportedException e) { throw new IOException(e); } } context.setStatus("Read " + map.getClass()); int index = 0; for (Cell kv: map) { context.write(row, new MapReduceExtendedCell(kv)); if (++index % 100 == 0) context.setStatus("Wrote " + index); } } }
protected void reduce( CellWritableComparable row, Iterable<Cell> kvs, Reducer<CellWritableComparable, Cell, ImmutableBytesWritable, Cell>.Context context) throws java.io.IOException, InterruptedException { int index = 0; for (Cell kv : kvs) { context.write(new ImmutableBytesWritable(CellUtil.cloneRow(kv)), new MapReduceExtendedCell(kv)); if (++index % 100 == 0) context.setStatus("Wrote " + index + " KeyValues, " + "and the rowkey whose is being wrote is " + Bytes.toString(kv.getRowArray())); } } }
@Override protected void reduce(Text arg0, Iterable<Text> arg1, Reducer<Text, Text, Text, Text>.Context context) throws IOException, InterruptedException { for (Text t : arg1) { tcs.add(t); } // If we're running in produciton context: if (context != null) { // Record progress: context.setStatus("Seen " + tcs.getTotalRecords() + " records, sent " + tcs.getTotalSentRecords() + "..."); // TODO Also pass to reducer output for cross-checking? } }
@Override protected void reduce(Text arg0, Iterable<Text> arg1, Reducer<Text, Text, Text, Text>.Context context) throws IOException, InterruptedException { for (Text t : arg1) { tcs.add(t); } // If we're running in produciton context: if (context != null) { // Record progress: context.setStatus("Seen " + tcs.getTotalRecords() + " records, sent " + tcs.getTotalSentRecords() + "..."); // TODO Also pass to reducer output for cross-checking? } }
protected void reduce(ImmutableBytesWritable row, java.lang.Iterable<KeyValue> kvs, org.apache.hadoop.mapreduce.Reducer<ImmutableBytesWritable, KeyValue, ImmutableBytesWritable, KeyValue>.Context context) throws java.io.IOException, InterruptedException { TreeSet<KeyValue> map = new TreeSet<KeyValue>(KeyValue.COMPARATOR); for (KeyValue kv: kvs) { map.add(kv.clone()); } context.setStatus("Read " + map.getClass()); int index = 0; for (KeyValue kv: map) { context.write(row, kv); if (index > 0 && index % 100 == 0) context.setStatus("Wrote " + index); } } }
public void reduce(LongWritable key, Iterable<NullWritable> values, Context context) throws IOException { if (context.getConfiguration().getBoolean(FAIL_REDUCE, false)) { throw new RuntimeException("Intentional reduce failure"); } context.setStatus("No worries"); } }
protected void reduce(KeyValueWritableComparable row, Iterable<KeyValue> kvs, Reducer<KeyValueWritableComparable, KeyValue, ImmutableBytesWritable, KeyValue>.Context context) throws java.io.IOException, InterruptedException { int index = 0; for (KeyValue kv : kvs) { context.write(new ImmutableBytesWritable(kv.getRowArray()), kv); if (++index % 100 == 0) context.setStatus("Wrote " + index + " KeyValues, " + "and the rowkey whose is being wrote is " + Bytes.toString(kv.getRowArray())); } } }
protected void reduce(KeyValueWritableComparable row, Iterable<KeyValue> kvs, Reducer<KeyValueWritableComparable, KeyValue, ImmutableBytesWritable, KeyValue>.Context context) throws java.io.IOException, InterruptedException { int index = 0; for (KeyValue kv : kvs) { context.write(new ImmutableBytesWritable(kv.getRowArray()), kv); if (++index % 100 == 0) context.setStatus("Wrote " + index + " KeyValues, " + "and the rowkey whose is being wrote is " + Bytes.toString(kv.getRowArray())); } } }
@Override public final void collect(K key, V value) throws IOException { try { context.setStatus("Writing Top K patterns for: " + key); context.write(key, value); } catch (InterruptedException e) { log.error("{}", e.toString()); } }
@Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long sum = 0; for (LongWritable value : values) { context.setStatus("Parallel Counting Reducer :" + key); sum += value.get(); } context.setStatus("Parallel Counting Reducer: " + key + " => " + sum); context.write(key, new LongWritable(sum)); } }
public void reduce(LongWritable key, Iterable<NullWritable> values, Context context) throws IOException { if (context.getConfiguration().getBoolean(FAIL_REDUCE, false)) { throw new RuntimeException("Intentional reduce failure"); } context.setStatus("No worries"); } }
@Override protected void cleanup(Context context) throws IOException, InterruptedException { final String msg = "Slept for " + duration; LOG.info(msg); context.setStatus(msg); } }
@Override public void update(String status) { long curTime = System.currentTimeMillis(); if (curTime - time > PERIOD && context != null) { time = curTime; context.setStatus("Processing FPTree: " + status); } }
@Override protected void reduce(Text key, Iterable<TopKStringPatterns> values, Context context) throws IOException, InterruptedException { TopKStringPatterns patterns = new TopKStringPatterns(); for (TopKStringPatterns value : values) { context.setStatus("Aggregator Reducer: Selecting TopK patterns for: " + key); patterns = patterns.merge(value, maxHeapSize); } context.write(key, patterns); }
protected void reduce( CellWritableComparable row, Iterable<Cell> kvs, Reducer<CellWritableComparable, Cell, ImmutableBytesWritable, Cell>.Context context) throws java.io.IOException, InterruptedException { int index = 0; for (Cell kv : kvs) { context.write(new ImmutableBytesWritable(CellUtil.cloneRow(kv)), new MapReduceExtendedCell(kv)); if (++index % 100 == 0) context.setStatus("Wrote " + index + " KeyValues, " + "and the rowkey whose is being wrote is " + Bytes.toString(kv.getRowArray())); } } }