@Override public void append(E entity) throws IOException { avroAppender.append(entity); parquetAppender.append(entity); }
@Override public void open() throws IOException { avroAppender.open(); parquetAppender.open(); }
@Override public void open() throws IOException { CompressionCodecName codecName = CompressionCodecName.UNCOMPRESSED; if (enableCompression) { codecName = getCompressionCodecName(); } avroParquetWriter = new AvroParquetWriter<E>(fileSystem.makeQualified(path), schema, codecName, DEFAULT_ROW_GROUP_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE, ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED, conf); }
public void createParquetUserFile(FileSystem fs, Path parent) throws IOException { Path file = new Path(parent, UUID.randomUUID().toString() + ".parquet"); ParquetAppender<Record> appender = new ParquetAppender<Record>( fs, file, USER_SCHEMA, new Configuration(), Uncompressed); appender.open(); appender.append(USER); appender.close(); }
public DurableParquetAppender(FileSystem fs, Path path, Schema schema, Configuration conf, CompressionType compressionType) { this.fs = fs; this.path = path; this.schema = schema; this.avroPath = avroPath(path); this.avroAppender = new AvroAppender<E>( fs, avroPath, schema, CompressionType.Snappy); this.parquetAppender = new ParquetAppender<E>( fs, path, schema, conf, compressionType); }
@Override public long pos() throws IOException { // based on final Parquet file size, not temporary Avro size return parquetAppender.pos(); }
public void createParquetEventFile(FileSystem fs, Path parent) throws IOException { Path file = new Path(parent, UUID.randomUUID().toString() + ".parquet"); ParquetAppender<Record> appender = new ParquetAppender<Record>( fs, file, EVENT_SCHEMA, new Configuration(), Uncompressed); appender.open(); appender.append(EVENT); appender.close(); }
@VisibleForTesting @SuppressWarnings("unchecked") <E> FileAppender<E> newAppender(Path temp) { Format format = descriptor.getFormat(); if (Formats.PARQUET.equals(format)) { // by default, Parquet is not durable if (DescriptorUtil.isDisabled( FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) { return (FileAppender<E>) new DurableParquetAppender( fs, temp, schema, conf, descriptor.getCompressionType()); } else { return (FileAppender<E>) new ParquetAppender( fs, temp, schema, conf, descriptor.getCompressionType()); } } else if (Formats.AVRO.equals(format)) { return new AvroAppender<E>(fs, temp, schema, descriptor.getCompressionType()); } else if (Formats.CSV.equals(format) && DescriptorUtil.isEnabled(FileSystemProperties.ALLOW_CSV_PROP, descriptor)) { return new CSVAppender<E>(fs, temp, descriptor); } else { this.state = ReaderWriterState.ERROR; throw new UnknownFormatException("Unknown format " + descriptor); } }