@Override public Metrics metrics() { return writer.metrics(); }
public OrcFileAppender build() { OrcFile.WriterOptions options = OrcFile.writerOptions(conf); return new OrcFileAppender(schema, file, options, metadata); } }
@Override public void add(InternalRow datum) { int row = batch.size++; for(int c=0; c < converters.length; ++c) { converters[c].addValue(row, c, datum, batch.cols[c]); } if (batch.size == BATCH_SIZE) { writer.add(batch); batch.reset(); } }
public SparkOrcWriter(OrcFileAppender writer) { TypeDescription schema = writer.getSchema(); batch = schema.createRowBatch(BATCH_SIZE); this.writer = writer; converters = buildConverters(schema); }