@Override public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException { long cuboidID = rowKeySplitter.split(key.getBytes()); Cuboid cuboid = Cuboid.findForMandatory(cubeDesc, cuboidID); int fullKeySize = buildKey(cuboid, rowKeySplitter.getSplitBuffers()); outputKey.set(newKeyBuf.array(), 0, fullKeySize); String baseOutputPath = PathNameCuboidOld; if (cuboidID == baseCuboid) { baseOutputPath = PathNameCuboidBase; } mos.write(outputKey, value, generateFileName(baseOutputPath)); }
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes); initCuboid(cuboidId); ByteArray[] splits = rowKeySplitter.getSplitBuffers(); int offset = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].array(), splits[offset].offset(), splits[offset].length()); offset++; } return cuboidId; }
@Override public Iterator<Tuple2<ByteArray, Object[]>> call(final Tuple2<ByteArray, Object[]> tuple2) throws Exception { if (initialized == false) { synchronized (SparkCubingByLayer.class) { if (initialized == false) { init(); initialized = true; } } } byte[] key = tuple2._1().array(); long cuboidId = rowKeySplitter.parseCuboid(key); final List<Long> myChildren = cubeSegment.getCuboidScheduler().getSpanningCuboid(cuboidId); // if still empty or null if (myChildren == null || myChildren.size() == 0) { return EMTPY_ITERATOR.iterator(); } rowKeySplitter.split(key); final Cuboid parentCuboid = Cuboid.findForMandatory(cubeDesc, cuboidId); List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size()); for (Long child : myChildren) { Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child); ByteArray result = ndCuboidBuilder.buildKey2(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); tuples.add(new Tuple2<>(result, tuple2._2())); } return tuples.iterator(); } }
for (Long child : myChildren) { Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child); result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); outputKey.set(result.getSecond().array(), 0, result.getFirst()); context.write(outputKey, value);
RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); ByteArray[] splittedByteses = rowKeySplitter.getSplitBuffers(); int bufOffset = 0; int bodySplitOffset = rowKeySplitter.getBodySplitOffset();
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes, bytes.length); initCuboid(cuboidId); SplittedBytes[] splits = rowKeySplitter.getSplitBuffers(); int offset = 1; // skip cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].value, splits[offset].length); offset++; } return cuboidId; }
int keyLength = buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); outputKey.set(keyBuf, 0, keyLength); context.write(outputKey, value);
Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidID); SplittedBytes[] splittedByteses = rowKeySplitter.getSplitBuffers(); int bufOffset = 0; BytesUtil.writeLong(cuboidID, newKeyBuf, bufOffset, RowConstants.ROWKEY_CUBOIDID_LEN);
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes, bytes.length); initCuboid(cuboidId); SplittedBytes[] splits = rowKeySplitter.getSplitBuffers(); int offset = 1; // skip cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].value, splits[offset].length); offset++; } return cuboidId; }
@Override public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException { long cuboidID = rowKeySplitter.split(key.getBytes()); Cuboid cuboid = Cuboid.findForMandatory(cubeDesc, cuboidID); int fullKeySize = buildKey(cuboid, rowKeySplitter.getSplitBuffers()); outputKey.set(newKeyBuf.array(), 0, fullKeySize); String baseOutputPath = PathNameCuboidOld; if (cuboidID == baseCuboid) { baseOutputPath = PathNameCuboidBase; } mos.write(outputKey, value, generateFileName(baseOutputPath)); }
int keyLength = buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); outputKey.set(keyBuf, 0, keyLength); context.write(outputKey, value);
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes); initCuboid(cuboidId); ByteArray[] splits = rowKeySplitter.getSplitBuffers(); int offset = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].array(), splits[offset].offset(), splits[offset].length()); offset++; } return cuboidId; }
@Override public Iterator<Tuple2<ByteArray, Object[]>> call(final Tuple2<ByteArray, Object[]> tuple2) throws Exception { if (initialized == false) { synchronized (SparkCubingByLayer.class) { if (initialized == false) { init(); initialized = true; } } } byte[] key = tuple2._1().array(); long cuboidId = rowKeySplitter.parseCuboid(key); final List<Long> myChildren = cubeSegment.getCuboidScheduler().getSpanningCuboid(cuboidId); // if still empty or null if (myChildren == null || myChildren.size() == 0) { return EMTPY_ITERATOR.iterator(); } rowKeySplitter.split(key); final Cuboid parentCuboid = Cuboid.findForMandatory(cubeDesc, cuboidId); List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size()); for (Long child : myChildren) { Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child); ByteArray result = ndCuboidBuilder.buildKey2(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); tuples.add(new Tuple2<>(result, tuple2._2())); } return tuples.iterator(); } }
for (Long child : myChildren) { Cuboid childCuboid = Cuboid.findForMandatory(cubeDesc, child); result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); outputKey.set(result.getSecond().array(), 0, result.getFirst()); context.write(outputKey, value);
Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidID); SplittedBytes[] splittedByteses = rowKeySplitter.getSplitBuffers(); int bufOffset = 0; BytesUtil.writeLong(cuboidID, newKeyBuf, bufOffset, RowConstants.ROWKEY_CUBOIDID_LEN);
RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); ByteArray[] splittedByteses = rowKeySplitter.getSplitBuffers(); int bufOffset = 0; int bodySplitOffset = rowKeySplitter.getBodySplitOffset();