@Override Object extractValue(Block block, int position) { return varcharType.getSlice(block, position); }
@Override Object extractValue(Block block, int position) { return varcharType.getSlice(block, position); }
@Override public void deserialize(Block block, int index, EvaluateClassifierPredictionsState state) { Slice slice = VARCHAR.getSlice(block, index); Map<String, Map<String, Integer>> jsonState; try { jsonState = OBJECT_MAPPER.readValue(slice.getBytes(), new TypeReference<Map<String, Map<String, Integer>>>() {}); } catch (IOException e) { throw new UncheckedIOException(e); } state.addMemoryUsage(slice.length()); state.getTruePositives().putAll(jsonState.getOrDefault(TRUE_POSITIVES, ImmutableMap.of())); state.getFalsePositives().putAll(jsonState.getOrDefault(FALSE_POSITIVES, ImmutableMap.of())); state.getFalseNegatives().putAll(jsonState.getOrDefault(FALSE_NEGATIVES, ImmutableMap.of())); } }
private static void checkMapValue(Map<DecoderColumnHandle, FieldValueProvider> decodedRow, DecoderTestColumnHandle handle, Map<String, String> expected) { Block actualBlock = getBlock(decodedRow, handle); assertEquals(actualBlock.getPositionCount(), expected.size() * 2); for (int i = 0; i < actualBlock.getPositionCount(); i += 2) { String actualKey = VARCHAR.getSlice(actualBlock, i).toStringUtf8(); String actualValue; if (actualBlock.isNull(i + 1)) { actualValue = null; } else { actualValue = VARCHAR.getSlice(actualBlock, i + 1).toStringUtf8(); } assertTrue(expected.containsKey(actualKey)); assertEquals(actualValue, expected.get(actualKey)); } }
private static void checkMapValue(Map<DecoderColumnHandle, FieldValueProvider> decodedRow, DecoderTestColumnHandle handle, Map<String, String> expected) { Block actualBlock = getBlock(decodedRow, handle); assertEquals(actualBlock.getPositionCount(), expected.size() * 2); for (int i = 0; i < actualBlock.getPositionCount(); i += 2) { String actualKey = VARCHAR.getSlice(actualBlock, i).toStringUtf8(); String actualValue; if (actualBlock.isNull(i + 1)) { actualValue = null; } else { actualValue = VARCHAR.getSlice(actualBlock, i + 1).toStringUtf8(); } assertTrue(expected.containsKey(actualKey)); assertEquals(actualValue, expected.get(actualKey)); } }
private void assertValue(Block rowBlock, int position, List<Object> row) { // null rows are handled by assertPositionValue requireNonNull(row, "row is null"); assertFalse(rowBlock.isNull(position)); SingleRowBlock singleRowBlock = (SingleRowBlock) rowBlock.getObject(position, Block.class); assertEquals(singleRowBlock.getPositionCount(), row.size()); for (int i = 0; i < row.size(); i++) { Object fieldValue = row.get(i); if (fieldValue == null) { assertTrue(singleRowBlock.isNull(i)); } else { if (fieldValue instanceof Long) { assertEquals(BIGINT.getLong(singleRowBlock, i), ((Long) fieldValue).longValue()); } else if (fieldValue instanceof String) { assertEquals(VARCHAR.getSlice(singleRowBlock, i), utf8Slice((String) fieldValue)); } else { throw new IllegalArgumentException(); } } } }
private void assertValue(Block rowBlock, int position, List<Object> row) { // null rows are handled by assertPositionValue requireNonNull(row, "row is null"); assertFalse(rowBlock.isNull(position)); SingleRowBlock singleRowBlock = (SingleRowBlock) rowBlock.getObject(position, Block.class); assertEquals(singleRowBlock.getPositionCount(), row.size()); for (int i = 0; i < row.size(); i++) { Object fieldValue = row.get(i); if (fieldValue == null) { assertTrue(singleRowBlock.isNull(i)); } else { if (fieldValue instanceof Long) { assertEquals(BIGINT.getLong(singleRowBlock, i), ((Long) fieldValue).longValue()); } else if (fieldValue instanceof String) { assertEquals(VARCHAR.getSlice(singleRowBlock, i), utf8Slice((String) fieldValue)); } else { throw new IllegalArgumentException(); } } } }
String actualKey = VARCHAR.getSlice(elementBlock, i).toStringUtf8(); Long actualValue; if (elementBlock.isNull(i + 1)) {
private static void test(IntStream keyInputStream, Stream<String> valueInputStream, BlockComparator comparator, Iterator<String> outputIterator) { BlockBuilder keysBlockBuilder = BIGINT.createBlockBuilder(null, INPUT_SIZE); BlockBuilder valuesBlockBuilder = VARCHAR.createBlockBuilder(null, INPUT_SIZE); keyInputStream.forEach(x -> BIGINT.writeLong(keysBlockBuilder, x)); valueInputStream.forEach(x -> VARCHAR.writeString(valuesBlockBuilder, x)); TypedKeyValueHeap heap = new TypedKeyValueHeap(comparator, BIGINT, VARCHAR, OUTPUT_SIZE); heap.addAll(keysBlockBuilder, valuesBlockBuilder); BlockBuilder resultBlockBuilder = VARCHAR.createBlockBuilder(null, OUTPUT_SIZE); heap.popAll(resultBlockBuilder); Block resultBlock = resultBlockBuilder.build(); assertEquals(resultBlock.getPositionCount(), OUTPUT_SIZE); for (int i = 0; i < OUTPUT_SIZE; i++) { assertEquals(VARCHAR.getSlice(resultBlock, i).toStringUtf8(), outputIterator.next()); } } }
String actualKey = VARCHAR.getSlice(elementBlock, i).toStringUtf8(); Long actualValue; if (elementBlock.isNull(i + 1)) {
private static void test(IntStream keyInputStream, Stream<String> valueInputStream, BlockComparator comparator, Iterator<String> outputIterator) { BlockBuilder keysBlockBuilder = BIGINT.createBlockBuilder(null, INPUT_SIZE); BlockBuilder valuesBlockBuilder = VARCHAR.createBlockBuilder(null, INPUT_SIZE); keyInputStream.forEach(x -> BIGINT.writeLong(keysBlockBuilder, x)); valueInputStream.forEach(x -> VARCHAR.writeString(valuesBlockBuilder, x)); TypedKeyValueHeap heap = new TypedKeyValueHeap(comparator, BIGINT, VARCHAR, OUTPUT_SIZE); heap.addAll(keysBlockBuilder, valuesBlockBuilder); BlockBuilder resultBlockBuilder = VARCHAR.createBlockBuilder(null, OUTPUT_SIZE); heap.popAll(resultBlockBuilder); Block resultBlock = resultBlockBuilder.build(); assertEquals(resultBlock.getPositionCount(), OUTPUT_SIZE); for (int i = 0; i < OUTPUT_SIZE; i++) { assertEquals(VARCHAR.getSlice(resultBlock, i).toStringUtf8(), outputIterator.next()); } } }
@Test public void testEvaluateClassifierPredictions() { metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = metadata.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("evaluate_classifier_predictions", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); String output = VARCHAR.getSlice(block, 0).toStringUtf8(); List<String> parts = ImmutableList.copyOf(Splitter.on('\n').omitEmptyStrings().split(output)); assertEquals(parts.size(), 7, output); assertEquals(parts.get(0), "Accuracy: 1/2 (50.00%)"); }
private static boolean filter(int position, Block discountBlock, Block shipDateBlock, Block quantityBlock) { return !shipDateBlock.isNull(position) && VARCHAR.getSlice(shipDateBlock, position).compareTo(MIN_SHIP_DATE) >= 0 && !shipDateBlock.isNull(position) && VARCHAR.getSlice(shipDateBlock, position).compareTo(MAX_SHIP_DATE) < 0 && !discountBlock.isNull(position) && DOUBLE.getDouble(discountBlock, position) >= 0.05 && !discountBlock.isNull(position) && DOUBLE.getDouble(discountBlock, position) <= 0.07 && !quantityBlock.isNull(position) && DOUBLE.getDouble(quantityBlock, position) < 24; } }
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
private static boolean filter(int position, Block discountBlock, Block shipDateBlock, Block quantityBlock) { return !shipDateBlock.isNull(position) && VARCHAR.getSlice(shipDateBlock, position).compareTo(MIN_SHIP_DATE) >= 0 && !shipDateBlock.isNull(position) && VARCHAR.getSlice(shipDateBlock, position).compareTo(MAX_SHIP_DATE) < 0 && !discountBlock.isNull(position) && DOUBLE.getDouble(discountBlock, position) >= 0.05 && !discountBlock.isNull(position) && DOUBLE.getDouble(discountBlock, position) <= 0.07 && !quantityBlock.isNull(position) && DOUBLE.getDouble(quantityBlock, position) < 24; } }
ImmutableSet.of("a", "c").contains(VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii())));
ImmutableSet.of("a", "c").contains(VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii())));
ImmutableSet.of("a", "c").contains(VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii())));
(leftPosition, leftPage, rightPosition, rightPage) -> VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii().equals("a")));