public static void main(String... args) throws Exception { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Path destFile = new Path(args[0]); LzopCodec codec = new LzopCodec(); codec.setConf(config); InputStream is = codec.createInputStream(hdfs.open(destFile)); readFromProtoBuf(is); } }
public static Path compress(Path src, Configuration config) throws IOException { Path destFile = new Path( src.toString() + new LzopCodec().getDefaultExtension()); LzopCodec codec = new LzopCodec(); codec.setConf(config); FileSystem hdfs = FileSystem.get(config); InputStream is = null; OutputStream os = null; try { is = hdfs.open(src); os = codec.createOutputStream(hdfs.create(destFile)); IOUtils.copyBytes(is, os, config); } finally { IOUtils.closeStream(os); IOUtils.closeStream(is); } return destFile; }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); LzopCodec codec = new LzopCodec(); codec.setConf(config); Path srcFile = new Path(args[0]); Path restoredFile = new Path(args[0] + ".restored"); Path destFile = compress(srcFile, config); decompress(destFile, restoredFile, config); }
getIndexedLzoOutputStream(Configuration conf, Path path) throws IOException { LzopCodec codec = new LzopCodec(); codec.setConf(conf); codec.createIndexedOutputStream(fileOut, indexOut) : codec.createOutputStream(fileOut) );
private static void generateInput(Configuration config, File inputFile, Path input) throws IOException { FileSystem hdfs = FileSystem.get(config); OutputStream os = hdfs.create(input); LzopCodec codec = new LzopCodec(); codec.setConf(config); OutputStream lzopOutputStream = codec.createOutputStream(os); ThriftBlockWriter<Stock> writer = new ThriftBlockWriter<Stock>( lzopOutputStream, Stock.class); for (String line : FileUtils.readLines(inputFile)) { Stock stock = createStock(line); writer.write(stock); } writer.finish(); writer.close(); IOUtils.closeStream(os); }
public WorkerThread(Config config, FileSystemManager fileSystemManager, TimeUnit pollSleepUnit, int threadIndex) { this.config = config; this.fileSystemManager = fileSystemManager; this.pollSleepUnit = pollSleepUnit; this.setDaemon(true); this.setName(WorkerThread.class.getSimpleName() + "-" + threadIndex); if (config.isCreateLzopIndex()) { this.indexer = new LzoIndexer(config.getConfig()); this.lzopExt = new LzopCodec().getDefaultExtension(); } else { this.indexer = null; } }
private static void generateInput(Configuration config, File inputFile, Path input) throws IOException { FileSystem hdfs = FileSystem.get(config); OutputStream os = hdfs.create(input); LzopCodec codec = new LzopCodec(); codec.setConf(config); OutputStream lzopOutputStream = codec.createOutputStream(os); ProtobufBlockWriter<Stock> writer = new ProtobufBlockWriter<Stock>( lzopOutputStream, Stock.class); for (String line : FileUtils.readLines(inputFile)) { Stock stock = createStock(line); writer.write(stock); } writer.finish(); writer.close(); IOUtils.closeStream(os); }
public TestLzopCodecByteAtATime() { com.hadoop.compression.lzo.LzopCodec codec = new com.hadoop.compression.lzo.LzopCodec(); codec.setConf(new Configuration()); this.verifyCodec = codec; }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Path destFile = new Path(args[0]); LzopCodec codec = new LzopCodec(); codec.setConf(config); InputStream is = codec.createInputStream(hdfs.open(destFile)); readFromProtoBuf(is); } }
public TestLzopCodec() { com.hadoop.compression.lzo.LzopCodec codec = new com.hadoop.compression.lzo.LzopCodec(); codec.setConf(new Configuration()); this.verifyCodec = codec; }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Path destFile = new Path(args[0]); LzopCodec codec = new LzopCodec(); codec.setConf(config); InputStream is = codec.createInputStream(hdfs.open(destFile)); readFromProtoBuf(is); } }
public static void decompress(Path src, Path dest, Configuration config) throws IOException { LzopCodec codec = new LzopCodec(); codec.setConf(config); FileSystem hdfs = FileSystem.get(config); InputStream is = null; OutputStream os = null; try { is = codec.createInputStream(hdfs.open(src)); os = hdfs.create(dest); IOUtils.copyBytes(is, os, config); } finally { IOUtils.closeStream(os); IOUtils.closeStream(is); } } }