private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder, ColumnFamilyDescriptor hcd) throws IOException { return init(methodName, conf, builder, hcd, null); }
private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder, ColumnFamilyDescriptor hcd, MyStoreHook hook) throws IOException { return init(methodName, conf, builder, hcd, hook, false); }
private HStore init(String methodName, Configuration conf, ColumnFamilyDescriptor hcd) throws IOException { return init(methodName, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd); }
private void init(String methodName) throws IOException { init(methodName, TEST_UTIL.getConfiguration()); }
@Test public void testStoreUsesSearchEngineOverride() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName()); init(this.name.getMethodName(), conf); assertEquals(DummyStoreEngine.lastCreatedCompactor, this.store.storeEngine.getCompactor()); }
private MyStore initMyStore(String methodName, Configuration conf, MyStoreHook hook) throws IOException { return (MyStore) init(methodName, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(5).build(), hook); }
private HStore init(String methodName, Configuration conf) throws IOException { // some of the tests write 4 versions and then flush // (with HBASE-4241, lower versions are collected on flush) return init(methodName, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(4).build()); }
@Test public void testRefreshStoreFilesNotChanged() throws IOException { init(name.getMethodName()); assertEquals(0, this.store.getStorefilesCount()); // add some data, flush this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); flush(1); // add one more file addStoreFile(); HStore spiedStore = spy(store); // call first time after files changed spiedStore.refreshStoreFiles(); assertEquals(2, this.store.getStorefilesCount()); verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); // call second time spiedStore.refreshStoreFiles(); //ensure that replaceStoreFiles is not called if files are not refreshed verify(spiedStore, times(0)).replaceStoreFiles(null, null); }
/** * Test for HBASE-3492 - Test split on empty colfam (no store files). * * @throws IOException When the IO operations fail. */ @Test public void testSplitWithEmptyColFam() throws IOException { init(this.name.getMethodName()); assertFalse(store.getSplitPoint().isPresent()); store.getHRegion().forceSplit(null); assertFalse(store.getSplitPoint().isPresent()); store.getHRegion().clearSplit(); }
@Test public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception { final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle"; long anyValue = 10; // We'll check that it uses correct config and propagates it appropriately by going thru // the simplest "real" path I can find - "throttleCompaction", which just checks whether // a number we pass in is higher than some config value, inside compactionPolicy. Configuration conf = HBaseConfiguration.create(); conf.setLong(CONFIG_KEY, anyValue); init(name.getMethodName() + "-xml", conf); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HTD overrides XML. --anyValue; init(name.getMethodName() + "-htd", conf, TableDescriptorBuilder .newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.of(family)); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HCD overrides them both. --anyValue; init(name.getMethodName() + "-hcd", conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.newBuilder(family).setValue(CONFIG_KEY, Long.toString(anyValue)) .build()); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); }
@Test public void testSpaceQuotaChangeAfterReplacement() throws IOException { final TableName tn = TableName.valueOf(name.getMethodName()); init(name.getMethodName()); RegionSizeStoreImpl sizeStore = new RegionSizeStoreImpl(); HStoreFile sf1 = mockStoreFileWithLength(1024L); HStoreFile sf2 = mockStoreFileWithLength(2048L); HStoreFile sf3 = mockStoreFileWithLength(4096L); HStoreFile sf4 = mockStoreFileWithLength(8192L); RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setStartKey(Bytes.toBytes("a")) .setEndKey(Bytes.toBytes("b")).build(); // Compacting two files down to one, reducing size sizeStore.put(regionInfo, 1024L + 4096L); store.updateSpaceQuotaAfterFileReplacement( sizeStore, regionInfo, Arrays.asList(sf1, sf3), Arrays.asList(sf2)); assertEquals(2048L, sizeStore.getRegionSize(regionInfo).getSize()); // The same file length in and out should have no change store.updateSpaceQuotaAfterFileReplacement( sizeStore, regionInfo, Arrays.asList(sf2), Arrays.asList(sf2)); assertEquals(2048L, sizeStore.getRegionSize(regionInfo).getSize()); // Increase the total size used store.updateSpaceQuotaAfterFileReplacement( sizeStore, regionInfo, Arrays.asList(sf2), Arrays.asList(sf3)); assertEquals(4096L, sizeStore.getRegionSize(regionInfo).getSize()); RegionInfo regionInfo2 = RegionInfoBuilder.newBuilder(tn).setStartKey(Bytes.toBytes("b")) .setEndKey(Bytes.toBytes("c")).build(); store.updateSpaceQuotaAfterFileReplacement(sizeStore, regionInfo2, null, Arrays.asList(sf4)); assertEquals(8192L, sizeStore.getRegionSize(regionInfo2).getSize()); }
private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException { init(this.name.getMethodName(), TEST_UTIL.getConfiguration(), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build()); long currentTs = 100; long minTs = currentTs; // the extra cell won't be flushed to disk, // so the min of timerange will be different between memStore and hfile. for (int i = 0; i != (maxVersion + 1); ++i) { this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[])null), null); if (i == 1) { minTs = currentTs; } } flushStore(store, id++); Collection<HStoreFile> files = store.getStorefiles(); assertEquals(1, files.size()); HStoreFile f = files.iterator().next(); f.initReader(); StoreFileReader reader = f.getReader(); assertEquals(minTs, reader.timeRange.getMin()); assertEquals(currentTs, reader.timeRange.getMax()); }
long[] timestamps2 = new long[] {30,80}; init(this.name.getMethodName());
@Test public void testLowestModificationTime() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); // Initialize region init(name.getMethodName(), conf); int storeFileNum = 4; for (int i = 1; i <= storeFileNum; i++) { LOG.info("Adding some data for the store file #"+i); this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); flush(i); } // after flush; check the lowest time stamp long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); }
/** * Verify that compression and data block encoding are respected by the * Store.createWriterInTmp() method, used on store flush. */ @Test public void testCreateWriter() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family) .setCompressionType(Compression.Algorithm.GZ).setDataBlockEncoding(DataBlockEncoding.DIFF) .build(); init(name.getMethodName(), conf, hcd); // Test createWriterInTmp() StoreFileWriter writer = store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false); Path path = writer.getPath(); writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1))); writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2))); writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3))); writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4))); writer.close(); // Verify that compression and encoding settings are respected HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), true, conf); assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm()); assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding()); reader.close(); }
init(this.name.getMethodName());
/** * Getting data from memstore and files * @throws IOException */ @Test public void testGet_FromMemStoreAndFiles() throws IOException { init(this.name.getMethodName()); //Put data in memstore this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); //flush flush(1); //Add more data this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); //flush flush(2); //Add more data this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); //Get result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers); //Need to sort the result since multiple files Collections.sort(result, CellComparatorImpl.COMPARATOR); //Compare assertCheck(); }
init(this.name.getMethodName());
/** * Getting data from memstore only * @throws IOException */ @Test public void testGet_FromMemStoreOnly() throws IOException { init(this.name.getMethodName()); //Put data in memstore this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); //Get result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers); //Compare assertCheck(); }
@Test public void testRefreshStoreFiles() throws Exception { init(name.getMethodName());