private void enqueueFromDump(int index) { if (dumpIterators.get(index) != null && dumpIterators.get(index).hasNext()) { Pair<byte[], byte[]> pair = dumpIterators.get(index).next(); minHeap.offer(new SimpleEntry(pair.getFirst(), index)); Object[] metricValues = new Object[metrics.trueBitCount()]; measureCodec.decode(ByteBuffer.wrap(pair.getSecond()), metricValues); dumpCurrentValues.set(index, metricValues); } }
@Override public void doReduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (Text value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input); aggs.aggregate(input, needAggrMeasures); } aggs.collectStates(result); ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(key, outputValue); } }
/** * Re-encode with measures in Object[] format. * @param key * @param value * @return * @throws IOException */ public Pair<Text, Object[]> reEncode2(Text key, Text value) throws IOException { if (initialized == false) { throw new IllegalStateException("Not initialized"); } Object[] measureObjs = new Object[measureDescs.size()]; codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs); // re-encode measures if dictionary is used if (dictMeasures.size() > 0) { for (Pair<Integer, MeasureIngester> pair : dictMeasures) { int i = pair.getFirst(); MeasureIngester ingester = pair.getSecond(); measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts); } } return Pair.newPair(processKey(key), measureObjs); }
@Override public void doReduce(ByteArrayWritable key, Iterable<ByteArrayWritable> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (ByteArrayWritable value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(value.asBuffer(), input); aggs.aggregate(input); } aggs.collectStates(result); // output key outputKey.set(key.array(), key.offset(), key.length()); // output value ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(outputKey, outputValue); }
/** * Re-encode with both dimension and measure in encoded (Text) format. * @param key * @param value * @return * @throws IOException */ public Pair<Text, Text> reEncode(Text key, Text value) throws IOException { if (initialized == false) { throw new IllegalStateException("Not initialized"); } Object[] measureObjs = new Object[measureDescs.size()]; // re-encode measures if dictionary is used if (dictMeasures.size() > 0) { codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs); for (Pair<Integer, MeasureIngester> pair : dictMeasures) { int i = pair.getFirst(); MeasureIngester ingester = pair.getSecond(); measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts); } ByteBuffer valueBuf = codec.encode(measureObjs); textValue.set(valueBuf.array(), 0, valueBuf.position()); return Pair.newPair(processKey(key), textValue); } else { return Pair.newPair(processKey(key), value); } }
@Test public void basicTest() { MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") }; BufferedMeasureCodec codec = new BufferedMeasureCodec(descs); Double d = new Double(1.0); Long l = new Long(2); BigDecimal b = new BigDecimal("333.1234"); HLLCounter hllc = new HLLCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); BitmapCounter bitmap = RoaringBitmapCounterFactory.INSTANCE.newBitmap(); bitmap.add(123); bitmap.add(45678); bitmap.add(Integer.MAX_VALUE - 10); Object[] values = new Object[] { d, l, b, hllc, bitmap }; ByteBuffer buf = codec.encode(values); buf.flip(); System.out.println("size: " + buf.limit()); Object[] copy = new Object[values.length]; codec.decode(buf, copy); for (int i = 0; i < values.length; i++) { Object x = values[i]; Object y = copy[i]; assertEquals(x, y); } }
private void enqueueFromDump(int index) { if (dumpIterators.get(index) != null && dumpIterators.get(index).hasNext()) { Pair<byte[], byte[]> pair = dumpIterators.get(index).next(); minHeap.offer(new SimpleEntry(pair.getFirst(), index)); Object[] metricValues = new Object[metrics.trueBitCount()]; measureCodec.decode(ByteBuffer.wrap(pair.getSecond()), metricValues); dumpCurrentValues.set(index, metricValues); } }
@Override public void doReduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (Text value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input); aggs.aggregate(input, needAggrMeasures); } aggs.collectStates(result); ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(key, outputValue); } }
/** * Re-encode with measures in Object[] format. * @param key * @param value * @return * @throws IOException */ public Pair<Text, Object[]> reEncode2(Text key, Text value) throws IOException { if (initialized == false) { throw new IllegalStateException("Not initialized"); } Object[] measureObjs = new Object[measureDescs.size()]; codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs); // re-encode measures if dictionary is used if (dictMeasures.size() > 0) { for (Pair<Integer, MeasureIngester> pair : dictMeasures) { int i = pair.getFirst(); MeasureIngester ingester = pair.getSecond(); measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts); } } return Pair.newPair(processKey(key), measureObjs); }
@Override public void doReduce(ByteArrayWritable key, Iterable<ByteArrayWritable> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (ByteArrayWritable value : values) { if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) { logger.info("Handling value with ordinal (This is not KV number!): " + vcounter); } codec.decode(value.asBuffer(), input); aggs.aggregate(input); } aggs.collectStates(result); // output key outputKey.set(key.array(), key.offset(), key.length()); // output value ByteBuffer valueBuf = codec.encode(result); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(outputKey, outputValue); }
/** * Re-encode with both dimension and measure in encoded (Text) format. * @param key * @param value * @return * @throws IOException */ public Pair<Text, Text> reEncode(Text key, Text value) throws IOException { if (initialized == false) { throw new IllegalStateException("Not initialized"); } Object[] measureObjs = new Object[measureDescs.size()]; // re-encode measures if dictionary is used if (dictMeasures.size() > 0) { codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs); for (Pair<Integer, MeasureIngester> pair : dictMeasures) { int i = pair.getFirst(); MeasureIngester ingester = pair.getSecond(); measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts); } ByteBuffer valueBuf = codec.encode(measureObjs); textValue.set(valueBuf.array(), 0, valueBuf.position()); return Pair.newPair(processKey(key), textValue); } else { return Pair.newPair(processKey(key), value); } }