/** * If it's an LzopInputStream, return the compressed bytes read plus the offset. Otherwise, * default to superclass behavior. */ @Override public long getPosition() throws IOException { if (lzopIn_ != null) { return start_ + lzopIn_.getCompressedBytesRead(); } else { return super.getPosition(); } }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Path destFile = new Path(args[0]); LzopCodec codec = new LzopCodec(); codec.setConf(config); InputStream is = codec.createInputStream(hdfs.open(destFile)); readFromProtoBuf(is); } }
/** * @return true if "require.lzo.tests" system property is set or if native * lzo libraries are loaded. */ static public boolean okToRunLzoTests(Configuration conf) throws IOException { if (Boolean.parseBoolean(System.getProperty("require.lzo.tests"))) { return true; } try { return LzoCodec.isNativeLzoLoaded(conf); } catch (UnsatisfiedLinkError e) { LOG.warn("Unable to load native LZO, skipping tests that require it.", e); } return false; } }
public static Path compress(Path src, Configuration config) throws IOException { Path destFile = new Path( src.toString() + new LzopCodec().getDefaultExtension()); LzopCodec codec = new LzopCodec(); codec.setConf(config); FileSystem hdfs = FileSystem.get(config); InputStream is = null; OutputStream os = null; try { is = hdfs.open(src); os = codec.createOutputStream(hdfs.create(destFile)); IOUtils.copyBytes(is, os, config); } finally { IOUtils.closeStream(os); IOUtils.closeStream(is); } return destFile; }
private static void generateInput(Configuration config, File inputFile, Path input) throws IOException { FileSystem hdfs = FileSystem.get(config); OutputStream os = hdfs.create(input); LzopCodec codec = new LzopCodec(); codec.setConf(config); OutputStream lzopOutputStream = codec.createOutputStream(os); ThriftBlockWriter<Stock> writer = new ThriftBlockWriter<Stock>( lzopOutputStream, Stock.class); for (String line : FileUtils.readLines(inputFile)) { Stock stock = createStock(line); writer.write(stock); } writer.finish(); writer.close(); IOUtils.closeStream(os); }
public static Path compressAndIndex(Path file, Configuration conf) throws IOException { Configuration tmpConfig = new Configuration(conf); tmpConfig.setLong("dfs.block.size", 512); tmpConfig.setInt(LzoCodec.LZO_BUFFER_SIZE_KEY, 512); Path compressedFile = LzopFileReadWrite.compress(file, tmpConfig); compressedFile.getFileSystem(tmpConfig).delete(new Path( compressedFile.toString() + LzoIndex.LZO_INDEX_SUFFIX), false); new LzoIndexer(tmpConfig).index(compressedFile); LzoIndex index = LzoIndex .readIndex(compressedFile.getFileSystem(tmpConfig), compressedFile); for (int i = 0; i < index.getNumberOfBlocks(); i++) { System.out.println("block[" + i + "] = " + index.getPosition(i)); } Job job = new Job(conf); job.setInputFormatClass(LzoTextInputFormat.class); LzoTextInputFormat inputFormat = new LzoTextInputFormat(); TextInputFormat.setInputPaths(job, compressedFile); List<InputSplit> is = inputFormat.getSplits(job); System.out.println("input splits = " + is.size()); return compressedFile; }
getIndexedLzoOutputStream(Configuration conf, Path path) throws IOException { LzopCodec codec = new LzopCodec(); codec.setConf(conf); codec.createIndexedOutputStream(fileOut, indexOut) : codec.createOutputStream(fileOut) );
public TestLzopCodecByteAtATime() { com.hadoop.compression.lzo.LzopCodec codec = new com.hadoop.compression.lzo.LzopCodec(); codec.setConf(new Configuration()); this.verifyCodec = codec; }
public WorkerThread(Config config, FileSystemManager fileSystemManager, TimeUnit pollSleepUnit, int threadIndex) { this.config = config; this.fileSystemManager = fileSystemManager; this.pollSleepUnit = pollSleepUnit; this.setDaemon(true); this.setName(WorkerThread.class.getSimpleName() + "-" + threadIndex); if (config.isCreateLzopIndex()) { this.indexer = new LzoIndexer(config.getConfig()); this.lzopExt = new LzopCodec().getDefaultExtension(); } else { this.indexer = null; } }
? new LzoIndex() : LzoIndex.readIndex( file.getFileSystem(HadoopCompat.getConfiguration(job)), file); prevFile = file; if (index.isEmpty()) { long end = start + fileSplit.getLength(); long lzoStart = index.alignSliceStartToIndex(start, end); long lzoEnd = index.alignSliceEndToIndex(end, lzoSplitStatus.lzoFileStatus.getLen());
new LzoIndexer(jobConf).index(new Path(outputDirAsString));
destFs.delete(lzoIndexPath, false); indexer.index(destFile);
public TestLzoCodecByteAtATime() { com.hadoop.compression.lzo.LzoCodec codec = new com.hadoop.compression.lzo.LzoCodec(); codec.setConf(new Configuration()); this.verifyCodec = codec; }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Path destFile = new Path(args[0]); LzopCodec codec = new LzopCodec(); codec.setConf(config); InputStream is = codec.createInputStream(hdfs.open(destFile)); readFromProtoBuf(is); } }
private static void generateInput(Configuration config, File inputFile, Path input) throws IOException { FileSystem hdfs = FileSystem.get(config); OutputStream os = hdfs.create(input); LzopCodec codec = new LzopCodec(); codec.setConf(config); OutputStream lzopOutputStream = codec.createOutputStream(os); ProtobufBlockWriter<Stock> writer = new ProtobufBlockWriter<Stock>( lzopOutputStream, Stock.class); for (String line : FileUtils.readLines(inputFile)) { Stock stock = createStock(line); writer.write(stock); } writer.finish(); writer.close(); IOUtils.closeStream(os); }
public TestLzopCodec() { com.hadoop.compression.lzo.LzopCodec codec = new com.hadoop.compression.lzo.LzopCodec(); codec.setConf(new Configuration()); this.verifyCodec = codec; }
public TestLzoCodec() { com.hadoop.compression.lzo.LzoCodec codec = new com.hadoop.compression.lzo.LzoCodec(); codec.setConf(new Configuration()); this.verifyCodec = codec; }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Path destFile = new Path(args[0]); LzopCodec codec = new LzopCodec(); codec.setConf(config); InputStream is = codec.createInputStream(hdfs.open(destFile)); readFromProtoBuf(is); } }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); LzopCodec codec = new LzopCodec(); codec.setConf(config); Path srcFile = new Path(args[0]); Path restoredFile = new Path(args[0] + ".restored"); Path destFile = compress(srcFile, config); decompress(destFile, restoredFile, config); }
public static void decompress(Path src, Path dest, Configuration config) throws IOException { LzopCodec codec = new LzopCodec(); codec.setConf(config); FileSystem hdfs = FileSystem.get(config); InputStream is = null; OutputStream os = null; try { is = codec.createInputStream(hdfs.open(src)); os = hdfs.create(dest); IOUtils.copyBytes(is, os, config); } finally { IOUtils.closeStream(os); IOUtils.closeStream(is); } } }