private static void repeatToStringBlock(String value, int count, BlockBuilder blockBuilder) { for (int i = 0; i < count; i++) { VARCHAR.writeString(blockBuilder, value); } }
@Override void writeNextRandomValue(Random random, BlockBuilder builder) { varcharType.writeString(builder, nextString(random)); } }
public static Block createTestBlock() { BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 15); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "cherry"); VARCHAR.writeString(blockBuilder, "cherry"); VARCHAR.writeString(blockBuilder, "date"); return blockBuilder.build(); }
@OutputFunction(StandardTypes.VARCHAR) public static void output(@AggregationState EvaluateClassifierPredictionsState state, BlockBuilder out) { StringBuilder sb = new StringBuilder(); long correct = state.getTruePositives() .values() .stream() .reduce(0, Integer::sum); long total = correct + state.getFalsePositives().values().stream().reduce(0, Integer::sum); sb.append(format(Locale.US, "Accuracy: %d/%d (%.2f%%)\n", correct, total, 100.0 * correct / (double) total)); Set<String> labels = union(union(state.getTruePositives().keySet(), state.getFalsePositives().keySet()), state.getFalseNegatives().keySet()); for (String label : labels) { int truePositives = state.getTruePositives().getOrDefault(label, 0); int falsePositives = state.getFalsePositives().getOrDefault(label, 0); int falseNegatives = state.getFalseNegatives().getOrDefault(label, 0); sb.append(format(Locale.US, "Class '%s'\n", label)); sb.append(format(Locale.US, "Precision: %d/%d (%.2f%%)\n", truePositives, truePositives + falsePositives, 100.0 * truePositives / (double) (truePositives + falsePositives))); sb.append(format(Locale.US, "Recall: %d/%d (%.2f%%)\n", truePositives, truePositives + falseNegatives, 100.0 * truePositives / (double) (truePositives + falseNegatives))); } VARCHAR.writeString(out, sb.toString()); } }
public static Block createStringSequenceBlock(int start, int end) { BlockBuilder builder = VARCHAR.createBlockBuilder(null, 100); for (int i = start; i < end; i++) { VARCHAR.writeString(builder, String.valueOf(i)); } return builder.build(); }
private static Block createStringsBlock(List<ConnectorId> values) { VarcharType varchar = createUnboundedVarcharType(); BlockBuilder builder = varchar.createBlockBuilder(null, values.size()); for (ConnectorId value : values) { if (value == null) { builder.appendNull(); } else { varchar.writeString(builder, value.getCatalogName()); } } return builder.build(); } }
@Test public void testRoundTrip() { BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); }
@Override public Page getOutput() { if (!finishing) { return null; } QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().getQueryId()); checkState(queryInfo.getOutputStage().isPresent(), "Output stage is missing"); checkState(queryInfo.getOutputStage().get().getSubStages().size() == 1, "Expected one sub stage of explain node"); if (!hasFinalStageInfo(queryInfo.getOutputStage().get())) { return null; } String plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionRegistry, operatorContext.getSession(), verbose); BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1); VARCHAR.writeString(builder, plan); outputConsumed = true; return new Page(builder.build()); }
public static Block createStringDictionaryBlock(int start, int length) { checkArgument(length > 5, "block must have more than 5 entries"); int dictionarySize = length / 5; BlockBuilder builder = VARCHAR.createBlockBuilder(null, dictionarySize); for (int i = start; i < start + dictionarySize; i++) { VARCHAR.writeString(builder, String.valueOf(i)); } int[] ids = new int[length]; for (int i = 0; i < length; i++) { ids[i] = i % dictionarySize; } return new DictionaryBlock(builder.build(), ids); }
@Test public void testRoundTrip() { PagesSerde serde = new TestingPagesSerdeFactory().createPagesSerde(); BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 5); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); Page expectedPage = new Page(expectedBlock, expectedBlock, expectedBlock); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); writePages(serde, sliceOutput, expectedPage, expectedPage, expectedPage); List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, VARCHAR); Iterator<Page> pageIterator = readPages(serde, sliceOutput.slice().getInput()); assertPageEquals(types, pageIterator.next(), expectedPage); assertPageEquals(types, pageIterator.next(), expectedPage); assertPageEquals(types, pageIterator.next(), expectedPage); assertFalse(pageIterator.hasNext()); }
@Test private void testGetSizeInBytes() { int numEntries = 1000; VarcharType unboundedVarcharType = createUnboundedVarcharType(); VariableWidthBlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, numEntries, 20 * numEntries); for (int i = 0; i < numEntries; i++) { unboundedVarcharType.writeString(blockBuilder, String.valueOf(ThreadLocalRandom.current().nextLong())); } Block block = blockBuilder.build(); List<Block> splitQuarter = splitBlock(block, 4); long sizeInBytes = block.getSizeInBytes(); long quarter1size = splitQuarter.get(0).getSizeInBytes(); long quarter2size = splitQuarter.get(1).getSizeInBytes(); long quarter3size = splitQuarter.get(2).getSizeInBytes(); long quarter4size = splitQuarter.get(3).getSizeInBytes(); double expectedQuarterSizeMin = sizeInBytes * 0.2; double expectedQuarterSizeMax = sizeInBytes * 0.3; assertTrue(quarter1size > expectedQuarterSizeMin && quarter1size < expectedQuarterSizeMax, format("quarter1size is %s, should be between %s and %s", quarter1size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter2size > expectedQuarterSizeMin && quarter2size < expectedQuarterSizeMax, format("quarter2size is %s, should be between %s and %s", quarter2size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter3size > expectedQuarterSizeMin && quarter3size < expectedQuarterSizeMax, format("quarter3size is %s, should be between %s and %s", quarter3size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter4size > expectedQuarterSizeMin && quarter4size < expectedQuarterSizeMax, format("quarter4size is %s, should be between %s and %s", quarter4size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertEquals(quarter1size + quarter2size + quarter3size + quarter4size, sizeInBytes); }
@Setup public void setup() { MapType mapType = mapType(VARCHAR, BIGINT); blockBuilder = mapType.createBlockBuilder(null, POSITIONS); for (int position = 0; position < POSITIONS; position++) { BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); for (int i = 0; i < mapSize; i++) { VARCHAR.writeString(entryBuilder, String.valueOf(ThreadLocalRandom.current().nextInt())); BIGINT.writeLong(entryBuilder, ThreadLocalRandom.current().nextInt()); } blockBuilder.closeEntry(); } dataBlock = blockBuilder.build(); }
public static Block createStringsBlock(Iterable<String> values) { BlockBuilder builder = VARCHAR.createBlockBuilder(null, 100); for (String value : values) { if (value == null) { builder.appendNull(); } else { VARCHAR.writeString(builder, value); } } return builder.build(); }
@Test public void testVarcharSerializedSize() { BlockBuilder builder = VARCHAR.createBlockBuilder(null, 5); // empty page Page page = new Page(builder.build()); int pageSize = serializedSize(ImmutableList.of(VARCHAR), page); assertEquals(pageSize, 44); // page overhead // page with one value VARCHAR.writeString(builder, "alice"); page = new Page(builder.build()); int firstValueSize = serializedSize(ImmutableList.of(VARCHAR), page) - pageSize; assertEquals(firstValueSize, 4 + 5); // length + "alice" // page with two values VARCHAR.writeString(builder, "bob"); page = new Page(builder.build()); int secondValueSize = serializedSize(ImmutableList.of(VARCHAR), page) - (pageSize + firstValueSize); assertEquals(secondValueSize, 4 + 3); // length + "bob" (null shared with first entry) }
@Test public void testRoundTrip() { int positionCount = 40; // build dictionary BlockBuilder dictionaryBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(dictionaryBuilder, "alice"); VARCHAR.writeString(dictionaryBuilder, "bob"); VARCHAR.writeString(dictionaryBuilder, "charlie"); VARCHAR.writeString(dictionaryBuilder, "dave"); Block dictionary = dictionaryBuilder.build(); // build ids int[] ids = new int[positionCount]; for (int i = 0; i < 40; i++) { ids[i] = i % 4; } DictionaryBlock dictionaryBlock = new DictionaryBlock(dictionary, ids); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, dictionaryBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertTrue(actualBlock instanceof DictionaryBlock); DictionaryBlock actualDictionaryBlock = (DictionaryBlock) actualBlock; assertBlockEquals(VARCHAR, actualDictionaryBlock.getDictionary(), dictionary); for (int position = 0; position < actualDictionaryBlock.getPositionCount(); position++) { assertEquals(actualDictionaryBlock.getId(position), ids[position]); } assertEquals(actualDictionaryBlock.getDictionarySourceId(), dictionaryBlock.getDictionarySourceId()); }
private static void test(IntStream keyInputStream, Stream<String> valueInputStream, BlockComparator comparator, Iterator<String> outputIterator) { BlockBuilder keysBlockBuilder = BIGINT.createBlockBuilder(null, INPUT_SIZE); BlockBuilder valuesBlockBuilder = VARCHAR.createBlockBuilder(null, INPUT_SIZE); keyInputStream.forEach(x -> BIGINT.writeLong(keysBlockBuilder, x)); valueInputStream.forEach(x -> VARCHAR.writeString(valuesBlockBuilder, x)); TypedKeyValueHeap heap = new TypedKeyValueHeap(comparator, BIGINT, VARCHAR, OUTPUT_SIZE); heap.addAll(keysBlockBuilder, valuesBlockBuilder); BlockBuilder resultBlockBuilder = VARCHAR.createBlockBuilder(null, OUTPUT_SIZE); heap.popAll(resultBlockBuilder); Block resultBlock = resultBlockBuilder.build(); assertEquals(resultBlock.getPositionCount(), OUTPUT_SIZE); for (int i = 0; i < OUTPUT_SIZE; i++) { assertEquals(VARCHAR.getSlice(resultBlock, i).toStringUtf8(), outputIterator.next()); } } }
@OutputFunction(StandardTypes.VARCHAR) public static void output(SpatialPartitioningState state, BlockBuilder out) { if (state.getCount() == 0) { out.appendNull(); return; } List<Rectangle> samples = state.getSamples(); int partitionCount = state.getPartitionCount(); int maxItemsPerNode = (samples.size() + partitionCount - 1) / partitionCount; Rectangle envelope = state.getExtent(); // Add a small buffer on the right and upper sides Rectangle paddedExtent = new Rectangle(envelope.getXMin(), envelope.getYMin(), Math.nextUp(envelope.getXMax()), Math.nextUp(envelope.getYMax())); VARCHAR.writeString(out, KdbTreeUtils.toJson(buildKdbTree(maxItemsPerNode, paddedExtent, samples))); } }
createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
private List<Page> createInputPages() { ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(TYPES); LineItemGenerator lineItemGenerator = new LineItemGenerator(1, 1, 1); for (int j = 0; j < pagesCount; j++) { Iterator<LineItem> iterator = lineItemGenerator.iterator(); for (int i = 0; i < rowsPerPage; i++) { pageBuilder.declarePosition(); LineItem lineItem = iterator.next(); BIGINT.writeLong(pageBuilder.getBlockBuilder(0), lineItem.getOrderKey()); BIGINT.writeLong(pageBuilder.getBlockBuilder(1), lineItem.getDiscountPercent()); DOUBLE.writeDouble(pageBuilder.getBlockBuilder(2), lineItem.getDiscount()); VARCHAR.writeString(pageBuilder.getBlockBuilder(3), lineItem.getReturnFlag()); DOUBLE.writeDouble(pageBuilder.getBlockBuilder(4), lineItem.getExtendedPrice()); } pages.add(pageBuilder.build()); pageBuilder.reset(); } return pages.build(); }
@Override void writeNextRandomValue(Random random, BlockBuilder builder) { varcharType.writeString(builder, nextString(random)); } }