codec.createOutputStream(fileOut) );
public static Path compress(Path src, Configuration config) throws IOException { Path destFile = new Path( src.toString() + new LzopCodec().getDefaultExtension()); LzopCodec codec = new LzopCodec(); codec.setConf(config); FileSystem hdfs = FileSystem.get(config); InputStream is = null; OutputStream os = null; try { is = hdfs.open(src); os = codec.createOutputStream(hdfs.create(destFile)); IOUtils.copyBytes(is, os, config); } finally { IOUtils.closeStream(os); IOUtils.closeStream(is); } return destFile; }
private static void generateInput(Configuration config, File inputFile, Path input) throws IOException { FileSystem hdfs = FileSystem.get(config); OutputStream os = hdfs.create(input); LzopCodec codec = new LzopCodec(); codec.setConf(config); OutputStream lzopOutputStream = codec.createOutputStream(os); ThriftBlockWriter<Stock> writer = new ThriftBlockWriter<Stock>( lzopOutputStream, Stock.class); for (String line : FileUtils.readLines(inputFile)) { Stock stock = createStock(line); writer.write(stock); } writer.finish(); writer.close(); IOUtils.closeStream(os); }
private static void generateInput(Configuration config, File inputFile, Path input) throws IOException { FileSystem hdfs = FileSystem.get(config); OutputStream os = hdfs.create(input); LzopCodec codec = new LzopCodec(); codec.setConf(config); OutputStream lzopOutputStream = codec.createOutputStream(os); ProtobufBlockWriter<Stock> writer = new ProtobufBlockWriter<Stock>( lzopOutputStream, Stock.class); for (String line : FileUtils.readLines(inputFile)) { Stock stock = createStock(line); writer.write(stock); } writer.finish(); writer.close(); IOUtils.closeStream(os); }