@Override public Entry<byte[], MeasureAggregator[]> next() { // Use minimum heap to merge sort the keys, // also do aggregation for measures with same keys in different dumps resultAggrs.reset(); byte[] peekKey = minHeap.peek().getKey(); internalAggregate(); while (!minHeap.isEmpty() && bytesComparator.compare(peekKey, minHeap.peek().getKey()) == 0) { internalAggregate(); } return new SimpleEntry(peekKey, resultMeasureAggregators); }
@Override public void doReduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (Text value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input); aggs.aggregate(input, needAggrMeasures); } aggs.collectStates(result); ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(key, outputValue); } }
@Override public void doReduce(ByteArrayWritable key, Iterable<ByteArrayWritable> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (ByteArrayWritable value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(value.asBuffer(), input); aggs.aggregate(input); } aggs.collectStates(result); // output key outputKey.set(key.array(), key.offset(), key.length()); // output value ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(outputKey, outputValue); }
reuseAggrs.reset(); reuseAggrs.aggregate(metrics); do {
reuseAggrs.reset(); reuseAggrs.aggregate(metrics); do {
@Override public Entry<byte[], MeasureAggregator[]> next() { // Use minimum heap to merge sort the keys, // also do aggregation for measures with same keys in different dumps resultAggrs.reset(); byte[] peekKey = minHeap.peek().getKey(); internalAggregate(); while (!minHeap.isEmpty() && bytesComparator.compare(peekKey, minHeap.peek().getKey()) == 0) { internalAggregate(); } return new SimpleEntry(peekKey, resultMeasureAggregators); }
@Override public void doReduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (Text value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input); aggs.aggregate(input, needAggrMeasures); } aggs.collectStates(result); ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(key, outputValue); } }
@Override public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (Text value : values) { codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input); if (cuboidLevel > 0) { aggs.aggregate(input, needAggr); } else { aggs.aggregate(input); } } aggs.collectStates(result); valueBuf.clear(); codec.encode(result, valueBuf); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(key, outputValue); counter++; if (counter % BatchConstants.COUNTER_MAX == 0) { logger.info("Handled " + counter + " records!"); } }
@Override public void doReduce(ByteArrayWritable key, Iterable<ByteArrayWritable> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (ByteArrayWritable value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(value.asBuffer(), input); aggs.aggregate(input); } aggs.collectStates(result); // output key outputKey.set(key.array(), key.offset(), key.length()); // output value ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(outputKey, outputValue); }
reuseAggrs.reset(); reuseAggrs.aggregate(metrics); do {
reuseAggrs.reset(); reuseAggrs.aggregate(metrics); do {