RCFile.Writer writer = new RCFile.Writer(target.getFileSystem(conf), conf, target, () -> {}, codec); return new ExtendedRecordWriter()
final RCFile.Writer out = new RCFile.Writer(fs, task.getConfiguration(), file, task, codec);
/** {@inheritDoc} */ @Override public RecordWriter<WritableComparable, BytesRefArrayWritable> getRecordWriter( FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { Path outputPath = getWorkOutputPath(job); FileSystem fs = outputPath.getFileSystem(job); Path file = new Path(outputPath, name); CompressionCodec codec = null; if (getCompressOutput(job)) { Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job); } final RCFile.Writer out = new RCFile.Writer(fs, job, file, progress, codec); return new RecordWriter<WritableComparable, BytesRefArrayWritable>() { @Override public void close(Reporter reporter) throws IOException { out.close(); } @Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } }; }
/** * Create a RCFile output stream based on job configuration Uses user supplied compression flag * (rather than obtaining it from the Job Configuration). * * @param jc * Job configuration * @param fs * File System to create file in * @param file * Path to be created * @return output stream over the created rcfile */ public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file, boolean isCompressed) throws IOException { CompressionCodec codec = null; Class<?> codecClass = null; if (isCompressed) { codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc); } return new RCFile.Writer(fs, jc, file, null, codec); }
/** * Create a RCFile output stream based on job configuration Uses user supplied compression flag * (rather than obtaining it from the Job Configuration). * * @param jc * Job configuration * @param fs * File System to create file in * @param file * Path to be created * @return output stream over the created rcfile */ public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file, boolean isCompressed, Progressable progressable) throws IOException { CompressionCodec codec = null; if (isCompressed) { Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc); } return new RCFile.Writer(fs, jc, file, progressable, codec); }
RCFileOutputFormat.setColumnNumber(conf, expectedFieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, RCFile.createMetadata(new Text("apple"), new Text("block"),
columnNumber = key.getKeyBuffer().getColumnNumber(); RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, outPath, null, codec);
new RCFile.Writer(fs, conf, file, null, RCFile.createMetadata(new Text("apple"), new Text("block"),
columnNumber = key.getKeyBuffer().getColumnNumber(); RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, outPath, null, codec);
RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length);
columnNumber = key.getKeyBuffer().getColumnNumber(); RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, outPath, null, codec);
RCFileOutputFormat.setColumnNumber(cloneConf, bytesArray.length); cloneConf.setInt(HiveConf.ConfVars.HIVE_RCFILE_RECORD_INTERVAL.varname, intervalRecordCount); RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec);
/** {@inheritDoc} */ @Override public RecordWriter<WritableComparable, BytesRefArrayWritable> getRecordWriter( FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { Path outputPath = getWorkOutputPath(job); FileSystem fs = outputPath.getFileSystem(job); Path file = new Path(outputPath, name); CompressionCodec codec = null; if (getCompressOutput(job)) { Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job); } final RCFile.Writer out = new RCFile.Writer(fs, job, file, progress, codec); return new RecordWriter<WritableComparable, BytesRefArrayWritable>() { @Override public void close(Reporter reporter) throws IOException { out.close(); } @Override public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException { out.append(value); } }; }
cloneConf.setInt(HiveConf.ConfVars.HIVE_RCFILE_RECORD_INTERVAL.varname, intervalRecordCount); RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec);
columnNumber = key.getKeyBuffer().getColumnNumber(); RCFileOutputFormat.setColumnNumber(jc, columnNumber); outWriter = new RCFile.Writer(fs, jc, getOutPath(), null, codec);
cloneConf.setInt(HiveConf.ConfVars.HIVE_RCFILE_RECORD_INTERVAL.varname, intervalRecordCount); RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec);
/** * Create a RCFile output stream based on job configuration Uses user supplied compression flag * (rather than obtaining it from the Job Configuration). * * @param jc * Job configuration * @param fs * File System to create file in * @param file * Path to be created * @return output stream over the created rcfile */ public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file, boolean isCompressed, Progressable progressable) throws IOException { CompressionCodec codec = null; if (isCompressed) { Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc); } return new RCFile.Writer(fs, jc, file, progressable, codec); }
/** * Create a RCFile output stream based on job configuration Uses user supplied compression flag * (rather than obtaining it from the Job Configuration). * * @param jc * Job configuration * @param fs * File System to create file in * @param file * Path to be created * @return output stream over the created rcfile */ public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file, boolean isCompressed, Progressable progressable) throws IOException { CompressionCodec codec = null; if (isCompressed) { Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc); } return new RCFile.Writer(fs, jc, file, progressable, codec); }
private void writeTest(FileSystem fs, int count, Path file, byte[][] fieldsData, Configuration conf) throws IOException, SerDeException { cleanup(); RCFileOutputFormat.setColumnNumber(conf, fieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length); for (int i = 0; i < fieldsData.length; i++) { BytesRefWritable cu = null; cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length); bytes.set(i, cu); } for (int i = 0; i < count; i++) { writer.append(bytes); } writer.close(); long fileLen = fs.getFileStatus(file).getLen(); System.out.println("The file size of RCFile with " + bytes.size() + " number columns and " + count + " number rows is " + fileLen); }
private void writeRCFileTest(FileSystem fs, int rowCount, Path file, int columnNum, CompressionCodec codec) throws IOException { fs.delete(file, true); resetRandomGenerators(); RCFileOutputFormat.setColumnNumber(conf, columnNum); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, codec); byte[][] columnRandom; BytesRefArrayWritable bytes = new BytesRefArrayWritable(columnNum); columnRandom = new byte[columnNum][]; for (int i = 0; i < columnNum; i++) { BytesRefWritable cu = new BytesRefWritable(); bytes.set(i, cu); } for (int i = 0; i < rowCount; i++) { nextRandomRow(columnRandom, bytes); writer.append(bytes); } writer.close(); }