/** * Creates a parser for the bytes using the supplied content-type */ public static XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, BytesReference bytes, XContentType xContentType) throws IOException { Objects.requireNonNull(xContentType); Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { InputStream compressedInput = compressor.streamInput(bytes.streamInput()); if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } return XContentFactory.xContent(xContentType).createParser(xContentRegistry, deprecationHandler, compressedInput); } else { return xContentType.xContent().createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); } }
CompressibleBytesOutputStream(BytesStream bytesStreamOutput, boolean shouldCompress) throws IOException { this.bytesStreamOutput = bytesStreamOutput; this.shouldCompress = shouldCompress; if (shouldCompress) { this.stream = CompressorFactory.COMPRESSOR.streamOutput(Streams.flushOnCloseStream(bytesStreamOutput)); } else { this.stream = bytesStreamOutput; } }
@Nullable public static Compressor compressor(BytesReference bytes) { if (COMPRESSOR.isCompressed(bytes)) { // bytes should be either detected as compressed or as xcontent, // if we have bytes that can be either detected as compressed or // as a xcontent, we have a problem assert XContentHelper.xContentType(bytes) == null; return COMPRESSOR; } XContentType contentType = XContentHelper.xContentType(bytes); if (contentType == null) { if (isAncient(bytes)) { throw new IllegalStateException("unsupported compression: index was created before v2.0.0.beta1 and wasn't upgraded?"); } throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } return null; }
StreamOutput streamOutput = bStream; if (compress != null && compress && (compressThreshold == -1 || source.length() > compressThreshold)) { streamOutput = CompressorFactory.defaultCompressor().streamOutput(bStream); XContentType contentType = XContentFactory.xContentType(source); if (formatContentType != null && formatContentType != contentType) { XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, CompressorFactory.defaultCompressor().streamOutput(bStream)); builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(source)); builder.close(); } else { StreamOutput streamOutput = CompressorFactory.defaultCompressor().streamOutput(bStream); source.writeTo(streamOutput); streamOutput.close(); InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); if (compressedStreamInput.markSupported() == false) { compressedStreamInput = new BufferedInputStream(compressedStreamInput); StreamOutput streamOutput = CompressorFactory.defaultCompressor().streamOutput(bStream); XContentBuilder builder = XContentFactory.contentBuilder(formatContentType, streamOutput); builder.copyCurrentStructure(XContentFactory.xContent(contentType).createParser(compressedStreamInput));
/** * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using * {@link XContentBuilder#rawField(String, InputStream, XContentType)}. */ public static void writeRawField(String field, BytesReference source, XContentType xContentType, XContentBuilder builder, ToXContent.Params params) throws IOException { Objects.requireNonNull(xContentType); Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { try (InputStream compressedStreamInput = compressor.streamInput(source.streamInput())) { builder.rawField(field, compressedStreamInput, xContentType); } } else { try (InputStream stream = source.streamInput()) { builder.rawField(field, stream, xContentType); } } }
/** * Create a {@link CompressedXContent} out of a {@link ToXContent} instance. */ public CompressedXContent(ToXContent xcontent, XContentType type, ToXContent.Params params) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); OutputStream compressedStream = CompressorFactory.COMPRESSOR.streamOutput(bStream); CRC32 crc32 = new CRC32(); OutputStream checkedStream = new CheckedOutputStream(compressedStream, crc32); try (XContentBuilder builder = XContentFactory.contentBuilder(type, checkedStream)) { builder.startObject(); xcontent.toXContent(builder, params); builder.endObject(); } this.bytes = BytesReference.toBytes(bStream.bytes()); this.crc32 = (int) crc32.getValue(); assertConsistent(); }
public static Compressor compressor(ChannelBuffer buffer) { for (Compressor compressor : compressors) { if (compressor.isCompressed(buffer)) { return compressor; } } throw new NotCompressedException(); }
/** * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using * {@link XContentBuilder#rawField(String, InputStream)}. * @deprecated use {@link #writeRawField(String, BytesReference, XContentType, XContentBuilder, Params)} to avoid content type * auto-detection */ @Deprecated public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params) throws IOException { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { try (InputStream compressedStreamInput = compressor.streamInput(source.streamInput())) { builder.rawField(field, compressedStreamInput); } } else { try (InputStream stream = source.streamInput()) { builder.rawField(field, stream); } } }
protected BytesReference write(T obj) throws IOException { try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { if (compress) { try (StreamOutput compressedStreamOutput = CompressorFactory.COMPRESSOR.streamOutput(bytesStreamOutput)) { write(obj, compressedStreamOutput); } } else { write(obj, bytesStreamOutput); } return bytesStreamOutput.bytes(); } }
/** * @deprecated we don't compress lucene indexes anymore and rely on lucene codecs */ @Deprecated @Nullable public static Compressor compressor(IndexInput in) throws IOException { for (Compressor compressor : compressors) { if (compressor.isCompressed(in)) { return compressor; } } return null; }
private static BytesReference uncompress(BytesReference bytes, Compressor compressor) throws IOException { StreamInput compressed = compressor.streamInput(bytes.streamInput()); BytesStreamOutput bStream = new BytesStreamOutput(); Streams.copy(compressed, bStream); compressed.close(); return bStream.bytes(); } }
public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVersion) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) { stream.setVersion(nodeVersion); stream.writeBoolean(false); diff.writeTo(stream); } return bStream.bytes(); }
@Nullable public static Compressor compressor(BytesReference bytes) { if (COMPRESSOR.isCompressed(bytes)) { // bytes should be either detected as compressed or as xcontent, // if we have bytes that can be either detected as compressed or // as a xcontent, we have a problem assert XContentFactory.xContentType(bytes) == null; return COMPRESSOR; } XContentType contentType = XContentFactory.xContentType(bytes); if (contentType == null) { if (isAncient(bytes)) { throw new IllegalStateException("unsupported compression: index was created before v2.0.0.beta1 and wasn't upgraded?"); } throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } return null; }
throw new IllegalStateException(ex); streamInput = compressor.streamInput(streamInput);
public static BytesReference serializeFullClusterState(ClusterState clusterState, Version nodeVersion) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) { stream.setVersion(nodeVersion); stream.writeBoolean(true); clusterState.writeTo(stream); } return bStream.bytes(); }
@Nullable public static Compressor compressor(BytesReference bytes) { if (COMPRESSOR.isCompressed(bytes)) { // bytes should be either detected as compressed or as xcontent, // if we have bytes that can be either detected as compressed or // as a xcontent, we have a problem assert XContentHelper.xContentType(bytes) == null; return COMPRESSOR; } XContentType contentType = XContentHelper.xContentType(bytes); if (contentType == null) { if (isAncient(bytes)) { throw new IllegalStateException("unsupported compression: index was created before v2.0.0.beta1 and wasn't upgraded?"); } throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } return null; }
/** * Creates a parser based on the bytes provided * @deprecated use {@link #createParser(NamedXContentRegistry, DeprecationHandler, BytesReference, XContentType)} * to avoid content type auto-detection */ @Deprecated public static XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, BytesReference bytes) throws IOException { Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { InputStream compressedInput = compressor.streamInput(bytes.streamInput()); if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } final XContentType contentType = XContentFactory.xContentType(compressedInput); return XContentFactory.xContent(contentType).createParser(xContentRegistry, deprecationHandler, compressedInput); } else { return XContentFactory.xContent(xContentType(bytes)).createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); } }
public StreamOutput bytes(Compressor compressor) throws IOException { return compressor.streamOutput(bytes); }
@Nullable public static Compressor compressor(BytesReference bytes) { if (COMPRESSOR.isCompressed(bytes)) { // bytes should be either detected as compressed or as xcontent, // if we have bytes that can be either detected as compressed or // as a xcontent, we have a problem assert XContentHelper.xContentType(bytes) == null; return COMPRESSOR; } XContentType contentType = XContentHelper.xContentType(bytes); if (contentType == null) { if (isAncient(bytes)) { throw new IllegalStateException("unsupported compression: index was created before v2.0.0.beta1 and wasn't upgraded?"); } throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } return null; }
/** * Converts the given bytes into a map that is optionally ordered. The provided {@link XContentType} must be non-null. */ public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) throws ElasticsearchParseException { try { final XContentType contentType; InputStream input; Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { InputStream compressedStreamInput = compressor.streamInput(bytes.streamInput()); if (compressedStreamInput.markSupported() == false) { compressedStreamInput = new BufferedInputStream(compressedStreamInput); } input = compressedStreamInput; } else { input = bytes.streamInput(); } contentType = xContentType != null ? xContentType : XContentFactory.xContentType(input); try (InputStream stream = input) { return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), stream, ordered)); } } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); } }