@Override public Block apply(Block block) { ColumnarMap mapBlock = toColumnarMap(block); Block keysBlock = keyCoercer == null ? mapBlock.getKeysBlock() : keyCoercer.apply(mapBlock.getKeysBlock()); Block valuesBlock = valueCoercer == null ? mapBlock.getValuesBlock() : valueCoercer.apply(mapBlock.getValuesBlock()); boolean[] valueIsNull = new boolean[mapBlock.getPositionCount()]; int[] offsets = new int[mapBlock.getPositionCount() + 1]; for (int i = 0; i < mapBlock.getPositionCount(); i++) { valueIsNull[i] = mapBlock.isNull(i); offsets[i + 1] = offsets[i] + mapBlock.getEntryCount(i); } return ((MapType) toType).createBlockFromKeyValue(Optional.of(valueIsNull), offsets, keysBlock, valuesBlock); } }
fieldExtractor = block -> { ColumnarMap columnarMap = toColumnarMap(block); return ImmutableList.of(columnarMap.getKeysBlock(), columnarMap.getValuesBlock()); }; fieldBuilders = type.getTypeParameters().stream()
private void writeColumnarMap(ColumnarMap columnarMap) { // write nulls and lengths for (int position = 0; position < columnarMap.getPositionCount(); position++) { boolean present = !columnarMap.isNull(position); presentStream.writeBoolean(present); if (present) { nonNullValueCount++; lengthStream.writeLong(columnarMap.getEntryCount(position)); } } // write keys and value Block keysBlock = columnarMap.getKeysBlock(); if (keysBlock.getPositionCount() > 0) { keyWriter.writeBlock(keysBlock); valueWriter.writeBlock(columnarMap.getValuesBlock()); } }
private static void assertColumnarMap(Block block, Slice[][][] expectedValues) { ColumnarMap columnarMap = toColumnarMap(block); assertEquals(columnarMap.getPositionCount(), expectedValues.length); Block keysBlock = columnarMap.getKeysBlock(); Block valuesBlock = columnarMap.getValuesBlock(); int keysPosition = 0; int valuesPosition = 0; for (int position = 0; position < expectedValues.length; position++) { Slice[][] expectedMap = expectedValues[position]; assertEquals(columnarMap.isNull(position), expectedMap == null); if (expectedMap == null) { assertEquals(columnarMap.getEntryCount(position), 0); continue; } assertEquals(columnarMap.getEntryCount(position), expectedMap.length); for (int i = 0; i < columnarMap.getEntryCount(position); i++) { Slice[] expectedEntry = expectedMap[i]; Slice expectedKey = expectedEntry[0]; assertBlockPosition(keysBlock, keysPosition, expectedKey); keysPosition++; Slice expectedValue = expectedEntry[1]; assertBlockPosition(valuesBlock, valuesPosition, expectedValue); valuesPosition++; } } }
private static ColumnarMap toColumnarMap(RunLengthEncodedBlock rleBlock) { ColumnarMap columnarMap = toColumnarMap(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int entryCount = columnarMap.getEntryCount(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * entryCount; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * entryCount]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < entryCount; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarMap( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); }
private static ColumnarMap toColumnarMap(DictionaryBlock dictionaryBlock) { ColumnarMap columnarMap = toColumnarMap(dictionaryBlock.getDictionary()); // build new offsets int[] offsets = new int[dictionaryBlock.getPositionCount() + 1]; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); offsets[position + 1] = offsets[position] + columnarMap.getEntryCount(dictionaryId); } // reindex dictionary int[] dictionaryIds = new int[offsets[dictionaryBlock.getPositionCount()]]; int nextDictionaryIndex = 0; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); int entryCount = columnarMap.getEntryCount(dictionaryId); // adjust to the element block start offset int startOffset = columnarMap.getOffset(dictionaryId) - columnarMap.getOffset(0); for (int entryIndex = 0; entryIndex < entryCount; entryIndex++) { dictionaryIds[nextDictionaryIndex] = startOffset + entryIndex; nextDictionaryIndex++; } } return new ColumnarMap( dictionaryBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); }
fieldExtractor = block -> { ColumnarMap columnarMap = toColumnarMap(block); return ImmutableList.of(columnarMap.getKeysBlock(), columnarMap.getValuesBlock()); }; fieldBuilders = type.getTypeParameters().stream()
private void writeColumnarMap(ColumnarMap columnarMap) { // write nulls and lengths for (int position = 0; position < columnarMap.getPositionCount(); position++) { boolean present = !columnarMap.isNull(position); presentStream.writeBoolean(present); if (present) { nonNullValueCount++; lengthStream.writeLong(columnarMap.getEntryCount(position)); } } // write keys and value Block keysBlock = columnarMap.getKeysBlock(); if (keysBlock.getPositionCount() > 0) { keyWriter.writeBlock(keysBlock); valueWriter.writeBlock(columnarMap.getValuesBlock()); } }
private static ColumnarMap toColumnarMap(RunLengthEncodedBlock rleBlock) { ColumnarMap columnarMap = toColumnarMap(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int entryCount = columnarMap.getEntryCount(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * entryCount; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * entryCount]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < entryCount; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarMap( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); }
private static ColumnarMap toColumnarMap(DictionaryBlock dictionaryBlock) { ColumnarMap columnarMap = toColumnarMap(dictionaryBlock.getDictionary()); // build new offsets int[] offsets = new int[dictionaryBlock.getPositionCount() + 1]; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); offsets[position + 1] = offsets[position] + columnarMap.getEntryCount(dictionaryId); } // reindex dictionary int[] dictionaryIds = new int[offsets[dictionaryBlock.getPositionCount()]]; int nextDictionaryIndex = 0; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); int entryCount = columnarMap.getEntryCount(dictionaryId); // adjust to the element block start offset int startOffset = columnarMap.getOffset(dictionaryId) - columnarMap.getOffset(0); for (int entryIndex = 0; entryIndex < entryCount; entryIndex++) { dictionaryIds[nextDictionaryIndex] = startOffset + entryIndex; nextDictionaryIndex++; } } return new ColumnarMap( dictionaryBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); }