public static Slice toJson(JsonGeneratorWriter writer, ConnectorSession session, Block block) { try { SliceOutput output = new DynamicSliceOutput(40); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeStartArray(); for (int i = 0; i < block.getPositionCount(); i++) { writer.writeJsonValue(jsonGenerator, block, i, session); } jsonGenerator.writeEndArray(); } return output.slice(); } catch (IOException e) { throwIfUnchecked(e); throw new RuntimeException(e); } } }
@Override public void encodeColumn(Block block, SliceOutput output, EncodeOutput encodeOutput) { for (int position = 0; position < block.getPositionCount(); position++) { if (block.isNull(position)) { output.writeBytes(nullSequence); } else { Slice slice = type.getSlice(block, position); byte[] data = slice.getBytes(); slice = Slices.wrappedBuffer(base64Encoder.encode(data)); output.writeBytes(slice); } encodeOutput.closeEntry(); } }
@Override public final void encodeValueInto(Block block, int position, SliceOutput output) { buffer.reset(); encodeValue(block, position, buffer); // structural types nested in structural types are length prefixed Slice slice = buffer.slice(); output.writeInt(Integer.reverseBytes(slice.length())); output.writeBytes(slice); }
private static void writeLengthPrefixedString(SliceOutput output, String string) { byte[] bytes = string.getBytes(UTF_8); output.writeInt(bytes.length); output.writeBytes(bytes); } }
public Slice serialize() { compact(); int requiredBytes = SizeOf.SIZE_OF_BYTE + // format SizeOf.SIZE_OF_INT + // max buckets SizeOf.SIZE_OF_INT + // entry count SizeOf.SIZE_OF_DOUBLE * nextIndex + // values SizeOf.SIZE_OF_DOUBLE * nextIndex; // weights return Slices.allocate(requiredBytes) .getOutput() .appendByte(FORMAT_TAG) .appendInt(maxBuckets) .appendInt(nextIndex) .appendBytes(Slices.wrappedDoubleArray(values, 0, nextIndex)) .appendBytes(Slices.wrappedDoubleArray(weights, 0, nextIndex)) .getUnderlyingSlice(); }
@UsedByGeneratedCode public static Slice toJson(ObjectKeyProvider provider, JsonGeneratorWriter writer, ConnectorSession session, Block block) { try { Map<String, Integer> orderedKeyToValuePosition = new TreeMap<>(); for (int i = 0; i < block.getPositionCount(); i += 2) { String objectKey = provider.getObjectKey(block, i); orderedKeyToValuePosition.put(objectKey, i + 1); } SliceOutput output = new DynamicSliceOutput(40); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeStartObject(); for (Map.Entry<String, Integer> entry : orderedKeyToValuePosition.entrySet()) { jsonGenerator.writeFieldName(entry.getKey()); writer.writeJsonValue(jsonGenerator, block, entry.getValue(), session); } jsonGenerator.writeEndObject(); } return output.slice(); } catch (IOException e) { throwIfUnchecked(e); throw new RuntimeException(e); } } }
@ScalarOperator(CAST) @LiteralParameters("x") @SqlType(JSON) public static Slice castFromVarchar(@SqlType("varchar(x)") Slice value) { try { SliceOutput output = new DynamicSliceOutput(value.length() + 2); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeString(value.toStringUtf8()); } return output.slice(); } catch (IOException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, format("Cannot cast '%s' to %s", value.toStringUtf8(), JSON)); } }
@Override public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { SliceOutput output = new DynamicSliceOutput(64); writeBlock(blockEncodingSerde, output, block); String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); jsonGenerator.writeString(encoded); } }
@ScalarOperator(CAST) @SqlType(JSON) public static Slice castFromDate(ConnectorSession session, @SqlType(DATE) long value) { try { SliceOutput output = new DynamicSliceOutput(12); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeString(printDate((int) value)); } return output.slice(); } catch (IOException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, format("Cannot cast '%s' to %s", value, JSON)); } }
private static Slice decimalToJson(BigDecimal bigDecimal) { try { SliceOutput dynamicSliceOutput = new DynamicSliceOutput(32); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, dynamicSliceOutput)) { jsonGenerator.writeNumber(bigDecimal); } return dynamicSliceOutput.slice(); } catch (IOException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, format("Cannot cast '%f' to %s", bigDecimal, StandardTypes.JSON)); } }
@ScalarOperator(CAST) @SqlType(JSON) public static Slice castFromBoolean(@SqlType(BOOLEAN) boolean value) { try { SliceOutput output = new DynamicSliceOutput(5); try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) { jsonGenerator.writeBoolean(value); } return output.slice(); } catch (IOException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, format("Cannot cast '%s' to %s", value, JSON)); } }
VarcharType varcharType = (VarcharType) type; Slice value = (Slice) object; StringLiteral stringLiteral = new StringLiteral(value.toStringUtf8()); StringLiteral stringLiteral = new StringLiteral(((Slice) object).toStringUtf8()); return new Cast(stringLiteral, type.getDisplayName(), false, true); SliceOutput output = new DynamicSliceOutput(toIntExact(((Block) object).getSizeInBytes())); BlockSerdeUtil.writeBlock(blockEncodingSerde, output, (Block) object); object = output.slice(); FunctionCall fromBase64 = new FunctionCall(QualifiedName.of("from_base64"), ImmutableList.of(new StringLiteral(VarbinaryFunctions.toBase64((Slice) object).toStringUtf8()))); Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type); return new FunctionCall(QualifiedName.of(signature.getName()), ImmutableList.of(fromBase64));
while (idx < replacement.length()) { byte nextByte = replacement.getByte(idx); if (nextByte == '$') { idx++; if (idx == replacement.length()) { // not using checkArgument because `.toStringUtf8` is expensive throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: " + replacement.toStringUtf8()); nextByte = replacement.getByte(idx); throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: unknown group { " + new String(groupName, StandardCharsets.UTF_8) + " }"); backref = nextByte - '0'; if (backref < 0 || backref > 9) { // not using checkArgument because `.toStringUtf8` is expensive throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Illegal replacement sequence: " + replacement.toStringUtf8()); int end = region.end[backref]; if (beg != -1 && end != -1) { // the specific group doesn't exist in the current match, skip result.appendBytes(source.slice(beg, end - beg)); nextByte = replacement.getByte(idx); result.appendByte(nextByte); idx++;
public static Slice regexpReplace(@SqlType("varchar(x)") Slice source, @SqlType(JoniRegexpType.NAME) Regex pattern, @SqlType("varchar(y)") Slice replacement) Matcher matcher = pattern.matcher(source.getBytes()); SliceOutput sliceOutput = new DynamicSliceOutput(source.length() + replacement.length() * 5); Slice sliceBetweenReplacements = source.slice(lastEnd, matcher.getBegin() - lastEnd); lastEnd = matcher.getEnd(); sliceOutput.appendBytes(sliceBetweenReplacements); appendReplacement(sliceOutput, source, pattern, matcher.getEagerRegion(), replacement); sliceOutput.appendBytes(source.slice(lastEnd, source.length() - lastEnd)); return sliceOutput.slice();
@ScalarFunction @LiteralParameters("x") @SqlType(StandardTypes.JSON) public static Slice jsonParse(@SqlType("varchar(x)") Slice slice) { // cast(json_parse(x) AS t)` will be optimized into `$internal$json_string_to_array/map/row_cast` in ExpressionOptimizer // If you make changes to this function (e.g. use parse JSON string into some internal representation), // make sure `$internal$json_string_to_array/map/row_cast` is changed accordingly. try (JsonParser parser = createJsonParser(JSON_FACTORY, slice)) { byte[] in = slice.getBytes(); SliceOutput dynamicSliceOutput = new DynamicSliceOutput(in.length); SORTED_MAPPER.writeValue((OutputStream) dynamicSliceOutput, SORTED_MAPPER.readValue(parser, Object.class)); // nextToken() returns null if the input is parsed correctly, // but will throw an exception if there are trailing characters. parser.nextToken(); return dynamicSliceOutput.slice(); } catch (Exception e) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Cannot convert '%s' to JSON", slice.toStringUtf8())); } }
public SerializedPage serialize(Page page) { SliceOutput serializationBuffer = new DynamicSliceOutput(toIntExact((page.getSizeInBytes() + Integer.BYTES))); // block length is an int writeRawPage(page, serializationBuffer, blockEncodingSerde); if (!compressor.isPresent()) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } int maxCompressedLength = maxCompressedLength(serializationBuffer.size()); byte[] compressionBuffer = new byte[maxCompressedLength]; int actualCompressedLength = compressor.get().compress(serializationBuffer.slice().getBytes(), 0, serializationBuffer.size(), compressionBuffer, 0, maxCompressedLength); if (((1.0 * actualCompressedLength) / serializationBuffer.size()) > MINIMUM_COMPRESSION_RATIO) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } return new SerializedPage( Slices.copyOf(Slices.wrappedBuffer(compressionBuffer, 0, actualCompressedLength)), COMPRESSED, page.getPositionCount(), serializationBuffer.size()); }
@Setup public void setup() SliceOutput sliceOutput = new DynamicSliceOutput(sourceLength); Slice pattern; switch (patternString) { case ".*x.*": pattern = Slices.utf8Slice(".*x.*"); IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte); break; case ".*(x|y).*": pattern = Slices.utf8Slice(".*(x|y).*"); IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte); break; case "longdotstar": pattern = Slices.utf8Slice(".*coolfunctionname.*"); ThreadLocalRandom.current().ints(97, 123).limit(sourceLength).forEach(sliceOutput::appendByte); break; source = sliceOutput.slice(); checkState(source.length() == sourceLength, "source.length=%s, sourceLength=%s", source.length(), sourceLength);
@Override public void encodeValueInto(int depth, Block block, int position, SliceOutput output) { Slice slice = type.getSlice(block, position); byte[] data = slice.getBytes(); slice = Slices.wrappedBuffer(base64Encoder.encode(data)); output.writeBytes(slice); }
private static final Slice RCFILE_MAGIC = utf8Slice("RCF"); private static final int CURRENT_VERSION = 1; private static final String COLUMN_COUNT_METADATA_KEY = "hive.io.rcfile.column.number"; output.writeBytes(RCFILE_MAGIC); output.writeByte(CURRENT_VERSION); recordValidation(validation -> validation.setVersion((byte) CURRENT_VERSION)); output.writeBoolean(codecName.isPresent()); codecName.ifPresent(name -> writeLengthPrefixedString(output, utf8Slice(name))); recordValidation(validation -> validation.setCodecClassName(codecName)); output.writeInt(Integer.reverseBytes(metadata.size() + 2)); writeMetadataProperty(COLUMN_COUNT_METADATA_KEY, Integer.toString(types.size())); writeMetadataProperty(PRESTO_RCFILE_WRITER_VERSION_METADATA_KEY, PRESTO_RCFILE_WRITER_VERSION); output.writeLong(syncFirst); recordValidation(validation -> validation.setSyncFirst(syncFirst)); output.writeLong(syncSecond); recordValidation(validation -> validation.setSyncSecond(syncSecond));
public Slice serialize() { try (SliceOutput output = new DynamicSliceOutput(estimatedSerializedSize())) { output.appendByte(UNCOMPRESSED_FORMAT); Slice serializedHll = hll.serialize(); output.appendInt(serializedHll.length()); output.appendBytes(serializedHll); output.appendInt(maxHashes); output.appendInt(minhash.size()); for (long key : minhash.keySet()) { output.appendLong(key); } for (short value : minhash.values()) { output.appendShort(value); } return output.slice(); } catch (IOException e) { throw new UncheckedIOException(e); } }