public DurableParquetAppender(FileSystem fs, Path path, Schema schema, Configuration conf, CompressionType compressionType) { this.fs = fs; this.path = path; this.schema = schema; this.avroPath = avroPath(path); this.avroAppender = new AvroAppender<E>( fs, avroPath, schema, CompressionType.Snappy); this.parquetAppender = new ParquetAppender<E>( fs, path, schema, conf, compressionType); }
public void createParquetUserFile(FileSystem fs, Path parent) throws IOException { Path file = new Path(parent, UUID.randomUUID().toString() + ".parquet"); ParquetAppender<Record> appender = new ParquetAppender<Record>( fs, file, USER_SCHEMA, new Configuration(), Uncompressed); appender.open(); appender.append(USER); appender.close(); }
public void createParquetEventFile(FileSystem fs, Path parent) throws IOException { Path file = new Path(parent, UUID.randomUUID().toString() + ".parquet"); ParquetAppender<Record> appender = new ParquetAppender<Record>( fs, file, EVENT_SCHEMA, new Configuration(), Uncompressed); appender.open(); appender.append(EVENT); appender.close(); }
@VisibleForTesting @SuppressWarnings("unchecked") <E> FileAppender<E> newAppender(Path temp) { Format format = descriptor.getFormat(); if (Formats.PARQUET.equals(format)) { // by default, Parquet is not durable if (DescriptorUtil.isDisabled( FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) { return (FileAppender<E>) new DurableParquetAppender( fs, temp, schema, conf, descriptor.getCompressionType()); } else { return (FileAppender<E>) new ParquetAppender( fs, temp, schema, conf, descriptor.getCompressionType()); } } else if (Formats.AVRO.equals(format)) { return new AvroAppender<E>(fs, temp, schema, descriptor.getCompressionType()); } else if (Formats.CSV.equals(format) && DescriptorUtil.isEnabled(FileSystemProperties.ALLOW_CSV_PROP, descriptor)) { return new CSVAppender<E>(fs, temp, descriptor); } else { this.state = ReaderWriterState.ERROR; throw new UnknownFormatException("Unknown format " + descriptor); } }