public static void write(TableMetadata metadata, OutputFile outputFile) { try (OutputStreamWriter writer = new OutputStreamWriter( outputFile.location().endsWith(".gz") ? new GzipCompressorOutputStream(outputFile.create()) : outputFile.create())) { JsonGenerator generator = JsonUtil.factory().createGenerator(writer); generator.useDefaultPrettyPrinter(); toJson(metadata, generator); generator.flush(); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to write json to file: %s", outputFile); } }
@Override public PositionOutputStream createOrOverwrite(long ignored) throws IOException { return stream(file.createOrOverwrite()); }
OrcFileAppender(Schema schema, OutputFile file, OrcFile.WriterOptions options, Map<String,byte[]> metadata) { orcSchema = TypeConversion.toOrc(schema, columnIds); options.setSchema(orcSchema); path = new Path(file.location()); try { writer = OrcFile.createWriter(path, options); } catch (IOException e) { throw new RuntimeException("Can't create file " + path, e); } writer.addUserMetadata(COLUMN_NUMBERS_ATTRIBUTE, columnIds.serialize()); metadata.forEach( (key,value) -> writer.addUserMetadata(key, ByteBuffer.wrap(value))); }
@Override public PositionOutputStream create(long ignored) throws IOException { return stream(file.create()); }
public ManifestFile toManifestFile() { Preconditions.checkState(closed, "Cannot build ManifestFile, writer is not closed"); return new GenericManifestFile(location, file.toInputFile().getLength(), specId, snapshotId, addedFiles, existingFiles, deletedFiles, stats.summaries()); }
ManifestWriter(PartitionSpec spec, OutputFile file, long snapshotId) { this.location = file.location(); this.file = file; this.specId = spec.specId(); this.writer = newAppender(FileFormat.AVRO, spec, file); this.snapshotId = snapshotId; this.reused = new ManifestEntry(spec.partitionType()); this.stats = new PartitionSummary(spec); }
@SuppressWarnings("unchecked") private static <D> DataFileWriter<D> newAvroWriter( Schema schema, OutputFile file, Function<Schema, DatumWriter<?>> createWriterFunc, CodecFactory codec, Map<String, String> metadata) throws IOException { DataFileWriter<D> writer = new DataFileWriter<>( (DatumWriter<D>) createWriterFunc.apply(schema)); writer.setCodec(codec); for (Map.Entry<String, String> entry : metadata.entrySet()) { writer.setMeta(entry.getKey(), entry.getValue()); } // TODO: support overwrite return writer.create(schema, file.create()); } }