private static Block resourceGroupIdToBlock(ResourceGroupId resourceGroupId) { requireNonNull(resourceGroupId, "resourceGroupId is null"); List<String> segments = resourceGroupId.getSegments(); BlockBuilder blockBuilder = createUnboundedVarcharType().createBlockBuilder(null, segments.size()); for (String segment : segments) { createUnboundedVarcharType().writeSlice(blockBuilder, utf8Slice(segment)); } return blockBuilder.build(); }
public static Block createStringSequenceBlock(int start, int end) { BlockBuilder builder = VARCHAR.createBlockBuilder(null, 100); for (int i = start; i < end; i++) { VARCHAR.writeString(builder, String.valueOf(i)); } return builder.build(); }
private static Block createStringsBlock(List<ConnectorId> values) { VarcharType varchar = createUnboundedVarcharType(); BlockBuilder builder = varchar.createBlockBuilder(null, values.size()); for (ConnectorId value : values) { if (value == null) { builder.appendNull(); } else { varchar.writeString(builder, value.getCatalogName()); } } return builder.build(); } }
@Override public Page getOutput() { if (!finishing) { return null; } QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().getQueryId()); checkState(queryInfo.getOutputStage().isPresent(), "Output stage is missing"); checkState(queryInfo.getOutputStage().get().getSubStages().size() == 1, "Expected one sub stage of explain node"); if (!hasFinalStageInfo(queryInfo.getOutputStage().get())) { return null; } String plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionRegistry, operatorContext.getSession(), verbose); BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1); VARCHAR.writeString(builder, plan); outputConsumed = true; return new Page(builder.build()); }
public static Block createStringDictionaryBlock(int start, int length) { checkArgument(length > 5, "block must have more than 5 entries"); int dictionarySize = length / 5; BlockBuilder builder = VARCHAR.createBlockBuilder(null, dictionarySize); for (int i = start; i < start + dictionarySize; i++) { VARCHAR.writeString(builder, String.valueOf(i)); } int[] ids = new int[length]; for (int i = 0; i < length; i++) { ids[i] = i % dictionarySize; } return new DictionaryBlock(builder.build(), ids); }
@BeforeMethod(alwaysRun = true) public void setUp() throws Exception { type = VarcharType.createVarcharType(100); BlockBuilder blockBuilder = type.createBlockBuilder(null, 100, 10); valueStore = new ValueStore(100, blockBuilder); valueStoreSmall = new ValueStore(1, blockBuilder); block = BlockAssertions.createStringsBlock("a", "b", "c", "d"); }
@Test public void testGetElementPositionRandom() { TypedSet set = new TypedSet(VARCHAR, 1, FUNCTION_NAME); testGetElementPositionRandomFor(set); BlockBuilder emptyBlockBuilder = VARCHAR.createBlockBuilder(null, 3); TypedSet setWithPassedInBuilder = new TypedSet(VARCHAR, emptyBlockBuilder, 1, FUNCTION_NAME); testGetElementPositionRandomFor(setWithPassedInBuilder); }
public static Block createStringsBlock(Iterable<String> values) { BlockBuilder builder = VARCHAR.createBlockBuilder(null, 100); for (String value : values) { if (value == null) { builder.appendNull(); } else { VARCHAR.writeString(builder, value); } } return builder.build(); }
public Block extractAll(Slice source, long groupIndex) { Matcher matcher = re2jPattern.matcher(source); int group = toIntExact(groupIndex); validateGroup(group, matcher.groupCount()); BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 32); while (true) { if (!matcher.find()) { break; } Slice searchedGroup = matcher.group(group); if (searchedGroup == null) { blockBuilder.appendNull(); continue; } VARCHAR.writeSlice(blockBuilder, searchedGroup); } return blockBuilder.build(); }
private static void test(IntStream keyInputStream, Stream<String> valueInputStream, BlockComparator comparator, Iterator<String> outputIterator) { BlockBuilder keysBlockBuilder = BIGINT.createBlockBuilder(null, INPUT_SIZE); BlockBuilder valuesBlockBuilder = VARCHAR.createBlockBuilder(null, INPUT_SIZE); keyInputStream.forEach(x -> BIGINT.writeLong(keysBlockBuilder, x)); valueInputStream.forEach(x -> VARCHAR.writeString(valuesBlockBuilder, x)); TypedKeyValueHeap heap = new TypedKeyValueHeap(comparator, BIGINT, VARCHAR, OUTPUT_SIZE); heap.addAll(keysBlockBuilder, valuesBlockBuilder); BlockBuilder resultBlockBuilder = VARCHAR.createBlockBuilder(null, OUTPUT_SIZE); heap.popAll(resultBlockBuilder); Block resultBlock = resultBlockBuilder.build(); assertEquals(resultBlock.getPositionCount(), OUTPUT_SIZE); for (int i = 0; i < OUTPUT_SIZE; i++) { assertEquals(VARCHAR.getSlice(resultBlock, i).toStringUtf8(), outputIterator.next()); } } }
@Test public void testVarcharSerializedSize() { BlockBuilder builder = VARCHAR.createBlockBuilder(null, 5); // empty page Page page = new Page(builder.build()); int pageSize = serializedSize(ImmutableList.of(VARCHAR), page); assertEquals(pageSize, 44); // page overhead // page with one value VARCHAR.writeString(builder, "alice"); page = new Page(builder.build()); int firstValueSize = serializedSize(ImmutableList.of(VARCHAR), page) - pageSize; assertEquals(firstValueSize, 4 + 5); // length + "alice" // page with two values VARCHAR.writeString(builder, "bob"); page = new Page(builder.build()); int secondValueSize = serializedSize(ImmutableList.of(VARCHAR), page) - (pageSize + firstValueSize); assertEquals(secondValueSize, 4 + 3); // length + "bob" (null shared with first entry) }
public Block split(Slice source) { Matcher matcher = re2jPattern.matcher(source); BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 32); int lastEnd = 0; while (matcher.find()) { Slice slice = source.slice(lastEnd, matcher.start() - lastEnd); lastEnd = matcher.end(); VARCHAR.writeSlice(blockBuilder, slice); } VARCHAR.writeSlice(blockBuilder, source.slice(lastEnd, source.length() - lastEnd)); return blockBuilder.build(); }
@Test public void testRoundTrip() { PagesSerde serde = new TestingPagesSerdeFactory().createPagesSerde(); BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 5); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); Page expectedPage = new Page(expectedBlock, expectedBlock, expectedBlock); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); writePages(serde, sliceOutput, expectedPage, expectedPage, expectedPage); List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, VARCHAR); Iterator<Page> pageIterator = readPages(serde, sliceOutput.slice().getInput()); assertPageEquals(types, pageIterator.next(), expectedPage); assertPageEquals(types, pageIterator.next(), expectedPage); assertPageEquals(types, pageIterator.next(), expectedPage); assertFalse(pageIterator.hasNext()); }
private static BlockBuilder createBlockBuilderWithValues(Slice[][] expectedValues) { BlockBuilder blockBuilder = new ArrayBlockBuilder(VARCHAR, null, 100, 100); for (Slice[] expectedValue : expectedValues) { if (expectedValue == null) { blockBuilder.appendNull(); } else { BlockBuilder elementBlockBuilder = VARCHAR.createBlockBuilder(null, expectedValue.length); for (Slice v : expectedValue) { VARCHAR.writeSlice(elementBlockBuilder, v); } blockBuilder.appendStructure(elementBlockBuilder.build()); } } return blockBuilder; } }
@Test public void testNonPrimitiveSerialization() { AccumulatorStateFactory<SliceState> factory = StateCompiler.generateStateFactory(SliceState.class); AccumulatorStateSerializer<SliceState> serializer = StateCompiler.generateStateSerializer(SliceState.class); SliceState state = factory.createSingleState(); SliceState deserializedState = factory.createSingleState(); state.setSlice(null); BlockBuilder nullBlockBuilder = VARCHAR.createBlockBuilder(null, 1); serializer.serialize(state, nullBlockBuilder); Block nullBlock = nullBlockBuilder.build(); serializer.deserialize(nullBlock, 0, deserializedState); assertEquals(deserializedState.getSlice(), state.getSlice()); state.setSlice(utf8Slice("test")); BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1); serializer.serialize(state, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getSlice(), state.getSlice()); }
@ScalarFunction @SqlType("array(varchar)") public static Block oldArraySort(@SqlType("array(varchar)") Block block) { List<Integer> positions = Ints.asList(new int[block.getPositionCount()]); for (int i = 0; i < block.getPositionCount(); i++) { positions.set(i, i); } positions.sort((p1, p2) -> { //TODO: This could be quite slow, it should use parametric equals return VARCHAR.compareTo(block, p1, block, p2); }); BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, block.getPositionCount()); for (int position : positions) { VARCHAR.appendTo(block, position, blockBuilder); } return blockBuilder.build(); } }
public static BlockBuilder createBlockBuilderWithValues(Slice[][] expectedValues) { BlockBuilder blockBuilder = new ArrayBlockBuilder(VARCHAR, null, 100, 100); for (Slice[] expectedValue : expectedValues) { if (expectedValue == null) { blockBuilder.appendNull(); } else { BlockBuilder elementBlockBuilder = VARCHAR.createBlockBuilder(null, expectedValue.length); for (Slice v : expectedValue) { if (v == null) { elementBlockBuilder.appendNull(); } else { VARCHAR.writeSlice(elementBlockBuilder, v); } } blockBuilder.appendStructure(elementBlockBuilder.build()); } } return blockBuilder; } }
@Test public void testRoundTrip() { BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); }
public static Block createTestBlock() { BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 15); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "cherry"); VARCHAR.writeString(blockBuilder, "cherry"); VARCHAR.writeString(blockBuilder, "date"); return blockBuilder.build(); }
@ScalarFunction @SqlType("array(varchar)") public static Block oldArrayDistinct(@SqlType("array(varchar)") Block array) { if (array.getPositionCount() == 0) { return array; } TypedSet typedSet = new TypedSet(VARCHAR, array.getPositionCount(), "old_array_distinct"); BlockBuilder distinctElementBlockBuilder = VARCHAR.createBlockBuilder(null, array.getPositionCount()); for (int i = 0; i < array.getPositionCount(); i++) { if (!typedSet.contains(array, i)) { typedSet.add(array, i); VARCHAR.appendTo(array, i, distinctElementBlockBuilder); } } return distinctElementBlockBuilder.build(); } }