@Override public void init(FileSystem fs, Path path, Configuration conf, FSDataInputStream stream) throws IOException { this.conf = conf; this.path = path; this.fs = fs; this.fileLength = this.fs.getFileStatus(path).getLen(); String cellCodecClsName = initReader(stream); boolean compression = hasCompression(); if (compression) { // If compression is enabled, new dictionaries are created here. try { if (compressionContext == null) { compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), hasTagCompression()); } else { compressionContext.clear(); } } catch (Exception e) { throw new IOException("Failed to initialize CompressionContext", e); } } initAfterCompression(cellCodecClsName); }
@Override public byte[] uncompress(ByteString data, Enum dictIndex) throws IOException { return WALCellCodec.uncompressByteString(data, compressionContext.getDictionary(dictIndex)); } }
private boolean initializeCompressionContext(Configuration conf, Path path) throws IOException { boolean doCompress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); if (doCompress) { try { this.compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), conf.getBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true)); } catch (Exception e) { throw new IOException("Failed to initiate CompressionContext", e); } } return doCompress; }
private void doTest(boolean compressTags, boolean offheapKV) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
@Override public void init(FileSystem fs, Path path, Configuration conf) throws IOException { this.conf = conf; this.path = path; reader = new WALReader(fs, path, conf); this.fs = fs; // If compression is enabled, new dictionaries are created here. boolean compression = reader.isWALCompressionEnabled(); if (compression) { try { if (compressionContext == null) { compressionContext = new CompressionContext(LRUDictionary.class); } else { compressionContext.clear(); } } catch (Exception e) { throw new IOException("Failed to initialize CompressionContext", e); } } }
@SuppressWarnings("unused") @Test public void testConstructorsArePresent() throws Exception { // "testing" via the presence of these constructors IndexedWALEditCodec codec1 = new IndexedWALEditCodec(); IndexedWALEditCodec codec2 = new IndexedWALEditCodec(new Configuration(false), new CompressionContext(LRUDictionary.class, false, false)); } }
public CompressionContext(Class<? extends Dictionary> dictType, boolean recoveredEdits, boolean hasTagCompression) throws SecurityException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException { Constructor<? extends Dictionary> dictConstructor = dictType.getConstructor(); for (DictionaryIndex dictionaryIndex : DictionaryIndex.values()) { Dictionary newDictionary = dictConstructor.newInstance(); dictionaries.put(dictionaryIndex, newDictionary); } if(recoveredEdits) { getDictionary(DictionaryIndex.REGION).init(1); getDictionary(DictionaryIndex.TABLE).init(1); } else { getDictionary(DictionaryIndex.REGION).init(Short.MAX_VALUE); getDictionary(DictionaryIndex.TABLE).init(Short.MAX_VALUE); } getDictionary(DictionaryIndex.ROW).init(Short.MAX_VALUE); getDictionary(DictionaryIndex.FAMILY).init(Byte.MAX_VALUE); getDictionary(DictionaryIndex.QUALIFIER).init(Byte.MAX_VALUE); if (hasTagCompression) { tagCompressionContext = new TagCompressionContext(dictType, Short.MAX_VALUE); } }
@Override public void init(FileSystem fs, Path path, Configuration conf, FSDataInputStream stream) throws IOException { this.conf = conf; this.path = path; this.fs = fs; this.fileLength = this.fs.getFileStatus(path).getLen(); String cellCodecClsName = initReader(stream); boolean compression = hasCompression(); if (compression) { // If compression is enabled, new dictionaries are created here. try { if (compressionContext == null) { compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), hasTagCompression()); } else { compressionContext.clear(); } } catch (Exception e) { throw new IOException("Failed to initialize CompressionContext", e); } } initAfterCompression(cellCodecClsName); }
@Test public void testWithCompression() throws Exception { // get the FS ready to read/write the edit Path testDir = UTIL.getDataTestDir("TestReadWriteCustomEdits_withCompression"); Path testFile = new Path(testDir, "testfile"); FileSystem fs = UTIL.getTestFileSystem(); List<WALEdit> edits = getEdits(); CompressionContext compression = new CompressionContext(LRUDictionary.class, false, false); writeReadAndVerify(compression, fs, edits, testFile); }
private void writeCompressed(byte[] data, Enum dictIndex) throws IOException { Dictionary dict = compressionContext.getDictionary(dictIndex); assert dict != null; short dictIdx = dict.findEntry(data, 0, data.length); if (dictIdx == Dictionary.NOT_IN_DICTIONARY) { write(Dictionary.NOT_IN_DICTIONARY); StreamUtils.writeRawVInt32(this, data.length); write(data, 0, data.length); } else { StreamUtils.writeShort(this, dictIdx); } } }
try { if (this.compressionContext == null) { this.compressionContext = new CompressionContext(LRUDictionary.class); } else { this.compressionContext.clear();
@Test public void testWithCompression() throws Exception { // get the FS ready to read/write the edit Path testDir = UTIL.getDataTestDir("TestReadWriteCustomEdits_withCompression"); Path testFile = new Path(testDir, "testfile"); FileSystem fs = UTIL.getTestFileSystem(); List<WALEdit> edits = getEdits(); CompressionContext compression = new CompressionContext(LRUDictionary.class); WALEditCodec codec = WALEditCodec.create(UTIL.getConfiguration(), compression); writeReadAndVerify(codec, fs, edits, testFile); }
@Override public void write(Cell cell) throws IOException { // We first write the KeyValue infrastructure as VInts. StreamUtils.writeRawVInt32(out, KeyValueUtil.keyLength(cell)); StreamUtils.writeRawVInt32(out, cell.getValueLength()); // To support tags int tagsLength = cell.getTagsLength(); StreamUtils.writeRawVInt32(out, tagsLength); PrivateCellUtil.compressRow(out, cell, compression.getDictionary(CompressionContext.DictionaryIndex.ROW)); PrivateCellUtil.compressFamily(out, cell, compression.getDictionary(CompressionContext.DictionaryIndex.FAMILY)); PrivateCellUtil.compressQualifier(out, cell, compression.getDictionary(CompressionContext.DictionaryIndex.QUALIFIER)); // Write timestamp, type and value as uncompressed. StreamUtils.writeLong(out, cell.getTimestamp()); out.write(cell.getTypeByte()); PrivateCellUtil.writeValue(out, cell, cell.getValueLength()); if (tagsLength > 0) { if (compression.tagCompressionContext != null) { // Write tags using Dictionary compression PrivateCellUtil.compressTags(out, cell, compression.tagCompressionContext); } else { // Tag compression is disabled within the WAL compression. Just write the tags bytes as // it is. PrivateCellUtil.writeTags(out, cell, tagsLength); } } } }
public boolean initializeCompressionContext(Configuration conf, Path path) throws IOException { boolean doCompress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); if (doCompress) { try { this.compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), conf.getBoolean( CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true)); } catch (Exception e) { throw new IOException("Failed to initiate CompressionContext", e); } } return doCompress; }
compression.getDictionary(CompressionContext.DictionaryIndex.ROW)); checkLength(elemLen, Short.MAX_VALUE); pos = Bytes.putShort(backingArray, pos, (short)elemLen); compression.getDictionary(CompressionContext.DictionaryIndex.FAMILY)); checkLength(elemLen, Byte.MAX_VALUE); pos = Bytes.putByte(backingArray, pos, (byte)elemLen); compression.getDictionary(CompressionContext.DictionaryIndex.QUALIFIER)); pos += elemLen;
private void doTest(boolean compressTags, boolean offheapKV) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);