@Override public void append(E entity) throws IOException { avroAppender.append(entity); parquetAppender.append(entity); }
@Override public void flush() throws IOException { avroAppender.flush(); }
@Override public void open() throws IOException { avroAppender.open(); parquetAppender.open(); }
final FileSystem fs = getDFS(); final Path path = new Path("hdfs://" + auth + "/tmp/test.avro"); AvroAppender<String> appender = new AvroAppender<String>( fs, path, schema, Formats.AVRO.getDefaultCompressionType()); appender.open(); for (int i = 0; i < 10; i += 1) { String s = "string-" + i; appender.append(s); appender.flush(); appender.sync(); appender.append(s); 10, count(fs, path)); appender.close(); appender.cleanup();
public void createAvroEventFile(FileSystem fs, Path parent) throws IOException { Path file = new Path(parent, UUID.randomUUID().toString() + ".avro"); AvroAppender<Record> appender = new AvroAppender<Record>( fs, file, EVENT_SCHEMA, Uncompressed); appender.open(); appender.append(EVENT); appender.close(); }
public DurableParquetAppender(FileSystem fs, Path path, Schema schema, Configuration conf, CompressionType compressionType) { this.fs = fs; this.path = path; this.schema = schema; this.avroPath = avroPath(path); this.avroAppender = new AvroAppender<E>( fs, avroPath, schema, CompressionType.Snappy); this.parquetAppender = new ParquetAppender<E>( fs, path, schema, conf, compressionType); }
@Override public void sync() throws IOException { avroAppender.sync(); }
@Override public void open() throws IOException { writer = new ReflectDatumWriter<E>(); dataFileWriter = new DataFileWriter<E>(writer); if (enableCompression) { dataFileWriter.setCodec(getCodecFactory()); } out = fileSystem.create(path, true); dataFileWriter.create(schema, out); }
final FileSystem fs = getFS(); final Path path = new Path("file:/tmp/test.avro"); AvroAppender<String> appender = new AvroAppender<String>( fs, path, schema, Formats.AVRO.getDefaultCompressionType()); appender.open(); for (int i = 0; i < 10; i += 1) { String s = "string-" + i; appender.append(s); appender.flush(); appender.sync(); appender.append(s); 10, count(fs, path)); appender.close(); appender.cleanup();
public void createAvroUserFile(FileSystem fs, Path parent) throws IOException { Path file = new Path(parent, UUID.randomUUID().toString() + ".avro"); AvroAppender<Record> appender = new AvroAppender<Record>( fs, file, USER_SCHEMA, Uncompressed); appender.open(); appender.append(USER); appender.close(); }
@VisibleForTesting @SuppressWarnings("unchecked") <E> FileAppender<E> newAppender(Path temp) { Format format = descriptor.getFormat(); if (Formats.PARQUET.equals(format)) { // by default, Parquet is not durable if (DescriptorUtil.isDisabled( FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) { return (FileAppender<E>) new DurableParquetAppender( fs, temp, schema, conf, descriptor.getCompressionType()); } else { return (FileAppender<E>) new ParquetAppender( fs, temp, schema, conf, descriptor.getCompressionType()); } } else if (Formats.AVRO.equals(format)) { return new AvroAppender<E>(fs, temp, schema, descriptor.getCompressionType()); } else if (Formats.CSV.equals(format) && DescriptorUtil.isEnabled(FileSystemProperties.ALLOW_CSV_PROP, descriptor)) { return new CSVAppender<E>(fs, temp, descriptor); } else { this.state = ReaderWriterState.ERROR; throw new UnknownFormatException("Unknown format " + descriptor); } }
@Override public void sync() throws IOException { flush(); Hadoop.FSDataOutputStream.hsync.invoke(out); }