@Override protected boolean shouldSplit() { boolean force = region.shouldForceSplit(); boolean foundABigStore = false; for (HStore store : region.getStores()) { // If any of the stores are unable to split (eg they contain reference files) // then don't split if ((!store.canSplit())) { return false; } // Mark if any store is big enough if (store.getSize() > desiredMaxFileSize) { foundABigStore = true; } } return foundABigStore || force; }
@Override protected boolean shouldSplit() { float blockedReqRate = updateRate(); if (super.shouldSplit()) { return true; } if (EnvironmentEdgeManager.currentTime() < startTime + minAge) { return false; } for (HStore store: region.getStores()) { if (!store.canSplit()) { return false; } } if (blockedReqRate >= maxBlockedRequests) { if (LOG.isDebugEnabled()) { LOG.debug("Going to split region " + region.getRegionInfo().getRegionNameAsString() + " because it's too busy. Blocked Request rate: " + blockedReqRate); } return true; } return false; }
@Test public void testGetSplitPoint() throws IOException { ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); // For no stores, should not split assertFalse(policy.shouldSplit()); assertNull(policy.getSplitPoint()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint(); stores.add(mockStore); assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint())); // Add a bigger store. The split point should come from that one HStore mockStore2 = Mockito.mock(HStore.class); Mockito.doReturn(4000L).when(mockStore2).getSize(); Mockito.doReturn(true).when(mockStore2).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint(); stores.add(mockStore2); assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint())); }
@Test public void testConstantSizePolicyWithJitter() throws IOException { conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); htd.setMaxFileSize(Long.MAX_VALUE); boolean positiveJitter = false; ConstantSizeRegionSplitPolicy policy = null; while (!positiveJitter) { policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); positiveJitter = policy.positiveJitterRate(); } // add a store HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE assertFalse(policy.shouldSplit()); }
/** * Run through tests for a ConstantSizeRegionSplitPolicy * @param policy */ private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) { // For no stores, should not split assertFalse(policy.shouldSplit()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); assertTrue(policy.shouldSplit()); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); assertFalse(policy.shouldSplit()); // Reset splittability after above Mockito.doReturn(true).when(mockStore).canSplit(); // Set to a small size but turn on forceSplit. Should result in a split. Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); Mockito.doReturn(100L).when(mockStore).getSize(); assertTrue(policy.shouldSplit()); // Turn off forceSplit, should not split Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); // Clear families we added above stores.clear(); }
/** * Test setting up a customized split policy */ @Test public void testCustomPolicy() throws IOException { HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); myHtd.setValue(HTableDescriptor.SPLIT_POLICY, KeyPrefixRegionSplitPolicy.class.getName()); myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2)); HRegion myMockRegion = Mockito.mock(HRegion.class); Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor(); Mockito.doReturn(stores).when(myMockRegion).getStores(); HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint(); stores.add(mockStore); KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ab", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion) .getExplicitSplitPoint(); policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ef", Bytes.toString(policy.getSplitPoint())); }
@Test public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException { HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); myHtd.setValue(HTableDescriptor.SPLIT_POLICY, DelimitedKeyPrefixRegionSplitPolicy.class.getName()); myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ","); HRegion myMockRegion = Mockito.mock(HRegion.class); Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor(); Mockito.doReturn(stores).when(myMockRegion).getStores(); HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint(); stores.add(mockStore); DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ab", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion) .getExplicitSplitPoint(); policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("efg", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion) .getExplicitSplitPoint(); assertEquals("ijk", Bytes.toString(policy.getSplitPoint())); }
@Override protected boolean shouldSplit() { boolean force = region.shouldForceSplit(); boolean foundABigStore = false; // Get count of regions that have the same common table as this.region int tableRegionsCount = getCountOfCommonTableRegions(); // Get size to check long sizeToCheck = getSizeToCheck(tableRegionsCount); for (HStore store : region.getStores()) { // If any of the stores is unable to split (eg they contain reference files) // then don't split if (!store.canSplit()) { return false; } // Mark if any store is big enough long size = store.getSize(); if (size > sizeToCheck) { LOG.debug("ShouldSplit because " + store.getColumnFamilyName() + " size=" + StringUtils.humanSize(size) + ", sizeToCheck=" + StringUtils.humanSize(sizeToCheck) + ", regionsWithCommonTable=" + tableRegionsCount); foundABigStore = true; } } return foundABigStore || force; }
@Test public void testForceSplitRegionWithReference() throws IOException { htd.setMaxFileSize(1024L); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); stores.add(mockStore); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, IncreasingToUpperBoundRegionSplitPolicy.class.getName()); policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); }
Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore);
.filter(region -> scanner.containsKey(region.getRegionInfo().getEncodedName())) .forEach(r -> Assert.assertTrue("Contains an open file reference which can be split", !r.getStores().get(0).canSplit())); } finally { scanner.values().stream().forEach(s -> {
@Test public void testGetSplitPoint() throws IOException { ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); // For no stores, should not split assertFalse(policy.shouldSplit()); assertNull(policy.getSplitPoint()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint(); stores.add(mockStore); assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint())); // Add a bigger store. The split point should come from that one HStore mockStore2 = Mockito.mock(HStore.class); Mockito.doReturn(4000L).when(mockStore2).getSize(); Mockito.doReturn(true).when(mockStore2).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint(); stores.add(mockStore2); assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint())); }
@Test public void testConstantSizePolicyWithJitter() throws IOException { conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); htd.setMaxFileSize(Long.MAX_VALUE); boolean positiveJitter = false; ConstantSizeRegionSplitPolicy policy = null; while (!positiveJitter) { policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); positiveJitter = policy.positiveJitterRate(); } // add a store HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE assertFalse(policy.shouldSplit()); }
/** * Test setting up a customized split policy */ @Test public void testCustomPolicy() throws IOException { HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); myHtd.setValue(HTableDescriptor.SPLIT_POLICY, KeyPrefixRegionSplitPolicy.class.getName()); myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2)); HRegion myMockRegion = Mockito.mock(HRegion.class); Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor(); Mockito.doReturn(stores).when(myMockRegion).getStores(); HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint(); stores.add(mockStore); KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ab", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion) .getExplicitSplitPoint(); policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ef", Bytes.toString(policy.getSplitPoint())); }
/** * Run through tests for a ConstantSizeRegionSplitPolicy * @param policy */ private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) { // For no stores, should not split assertFalse(policy.shouldSplit()); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore); assertTrue(policy.shouldSplit()); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); assertFalse(policy.shouldSplit()); // Reset splittability after above Mockito.doReturn(true).when(mockStore).canSplit(); // Set to a small size but turn on forceSplit. Should result in a split. Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); Mockito.doReturn(100L).when(mockStore).getSize(); assertTrue(policy.shouldSplit()); // Turn off forceSplit, should not split Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); // Clear families we added above stores.clear(); }
@Test public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException { HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); myHtd.setValue(HTableDescriptor.SPLIT_POLICY, DelimitedKeyPrefixRegionSplitPolicy.class.getName()); myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ","); HRegion myMockRegion = Mockito.mock(HRegion.class); Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor(); Mockito.doReturn(stores).when(myMockRegion).getStores(); HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint(); stores.add(mockStore); DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ab", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion) .getExplicitSplitPoint(); policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("efg", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion) .getExplicitSplitPoint(); assertEquals("ijk", Bytes.toString(policy.getSplitPoint())); }
@Test public void testForceSplitRegionWithReference() throws IOException { htd.setMaxFileSize(1024L); // Add a store above the requisite size. Should split. HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); // Act as if there's a reference file or some other reason it can't split. // This should prevent splitting even though it's big enough. Mockito.doReturn(false).when(mockStore).canSplit(); stores.add(mockStore); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); ConstantSizeRegionSplitPolicy policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); Mockito.doReturn(false).when(mockRegion).shouldForceSplit(); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, IncreasingToUpperBoundRegionSplitPolicy.class.getName()); policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); assertFalse(policy.shouldSplit()); Mockito.doReturn(true).when(mockRegion).shouldForceSplit(); assertFalse(policy.shouldSplit()); }
Mockito.doReturn(true).when(mockStore).canSplit(); stores.add(mockStore);
.filter(region -> scanner.containsKey(region.getRegionInfo().getEncodedName())) .forEach(r -> Assert.assertTrue("Contains an open file reference which can be split", !r.getStores().get(0).canSplit())); } finally { scanner.values().stream().forEach(s -> {