public JsonORCFileWriter(LogFilePath logFilePath, CompressionCodec codec) throws IOException { Configuration conf = new Configuration(); Path path = new Path(logFilePath.getLogFilePath()); schema = schemaProvider.getSchema(logFilePath.getTopic(), logFilePath); List<TypeDescription> fieldTypes = schema.getChildren(); converters = new JsonConverter[fieldTypes.size()]; for (int c = 0; c < converters.length; ++c) { converters[c] = VectorColumnFiller.createConverter(fieldTypes .get(c)); } writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf) .compress(resolveCompression(codec)).setSchema(schema)); batch = schema.createRowBatch(); }
@Override public RecordWriter<NullWritable, V> getRecordWriter(TaskAttemptContext taskAttemptContext ) throws IOException { Configuration conf = taskAttemptContext.getConfiguration(); Path filename = getDefaultWorkFile(taskAttemptContext, EXTENSION); Writer writer = OrcFile.createWriter(filename, org.apache.orc.mapred.OrcOutputFormat.buildOptions(conf)); return new OrcMapreduceRecordWriter<V>(writer); }
@Override public void initialize(Map<String, Object> metaData) { try { Configuration conf = new Configuration(); // conf.set(OrcConf.BLOOM_FILTER_COLUMNS.getAttribute(), "tags"); processor = new OrcEntityProcessor(OrcFile.createWriter(new Path(filename), OrcFile.writerOptions(conf).setSchema(SCHEMA)), SCHEMA.createRowBatch()); } catch (IOException e) { throw new OsmosisRuntimeException(e); } }
OrcFileAppender(Schema schema, OutputFile file, OrcFile.WriterOptions options, Map<String,byte[]> metadata) { orcSchema = TypeConversion.toOrc(schema, columnIds); options.setSchema(orcSchema); path = new Path(file.location()); try { writer = OrcFile.createWriter(path, options); } catch (IOException e) { throw new RuntimeException("Can't create file " + path, e); } writer.addUserMetadata(COLUMN_NUMBERS_ATTRIBUTE, columnIds.serialize()); metadata.forEach( (key,value) -> writer.addUserMetadata(key, ByteBuffer.wrap(value))); }
public PentahoOrcRecordWriter( List<? extends IOrcOutputField> fields, TypeDescription schema, String filePath, Configuration conf ) { this.fields = fields; this.schema = schema; final AtomicInteger fieldNumber = new AtomicInteger(); //Mutable field count fields.forEach( field -> setOutputMeta( fieldNumber, field ) ); outputRowMetaAndData = new RowMetaAndData( outputRowMeta, new Object[ fieldNumber.get() ] ); try { S3NCredentialUtils.applyS3CredentialsToHadoopConfigurationIfNecessary( filePath, conf ); Path outputFile = new Path( S3NCredentialUtils.scrubFilePathIfNecessary( filePath ) ); writer = OrcFile.createWriter( outputFile, OrcFile.writerOptions( conf ) .setSchema( schema ) ); batch = schema.createRowBatch(); } catch ( IOException e ) { logger.error( e ); } //Write the addition metadata for the fields // new OrcMetaDataWriter( writer ).write( fields ); }
@Override public RecordWriter<NullWritable, V> getRecordWriter(FileSystem fileSystem, JobConf conf, String name, Progressable progressable ) throws IOException { Path path = getTaskOutputPath(conf, name); Writer writer = OrcFile.createWriter(path, buildOptions(conf).fileSystem(fileSystem)); return new OrcMapredRecordWriter<>(writer); } }
options = options.compress(compressionKind); Writer writer = OrcFile.createWriter(file, options);
Writer writer = OrcFile.createWriter(new Path(outputOrc), OrcFile.writerOptions(conf).setSchema(SCHEMA));
Configuration conf = new Configuration(); TypeDescription schema = getSchema(table.getSchema()); Writer writer = OrcFile.createWriter(new Path(this.path), OrcFile.writerOptions(conf).setSchema(schema)); VectorizedRowBatch batch = schema.createRowBatch();
output = createWriter(outputPath, options); } else if (!readerIsCompatible(schema, fileVersion, writerVersion, rowIndexStride, compression, userMetadata, input, reader)) {
conf.setBoolean(OrcConf.BLOCK_PADDING.getAttribute(), false); Writer writer = OrcFile.createWriter(new Path(outputOrc), OrcFile.writerOptions(conf).setSchema(schema));