/** * @param candidates pre-filtrate * @return filtered subset exclude all files above maxCompactSize * Also save all references. We MUST compact them */ protected ArrayList<HStoreFile> skipLargeFiles(ArrayList<HStoreFile> candidates, boolean mayUseOffpeak) { int pos = 0; while (pos < candidates.size() && !candidates.get(pos).isReference() && (candidates.get(pos).getReader().length() > comConf.getMaxCompactSize(mayUseOffpeak))) { ++pos; } if (pos > 0) { LOG.debug("Some files are too large. Excluding " + pos + " files from compaction candidates"); candidates.subList(0, pos).clear(); } return candidates; }
@Override public String toStringDetailed() { StringBuilder sb = new StringBuilder(); sb.append(this.getPath().toString()); sb.append(", isReference=").append(isReference()); sb.append(", isBulkLoadResult=").append(isBulkLoadResult()); if (isBulkLoadResult()) { sb.append(", bulkLoadTS="); OptionalLong bulkLoadTS = getBulkLoadTimestamp(); if (bulkLoadTS.isPresent()) { sb.append(bulkLoadTS.getAsLong()); } else { sb.append("NotPresent"); } } else { sb.append(", seqid=").append(getMaxSequenceId()); } sb.append(", majorCompaction=").append(isMajorCompactionResult()); return sb.toString(); }
protected HStoreFile createMockStoreFile(final long sizeInBytes, final long seqId) { HStoreFile mockSf = mock(HStoreFile.class); StoreFileReader reader = mock(StoreFileReader.class); String stringPath = "/hbase/testTable/regionA/" + RandomStringUtils.random(FILENAME_LENGTH, 0, 0, true, true, null, random); Path path = new Path(stringPath); when(reader.getSequenceID()).thenReturn(seqId); when(reader.getTotalUncompressedBytes()).thenReturn(sizeInBytes); when(reader.length()).thenReturn(sizeInBytes); when(mockSf.getPath()).thenReturn(path); when(mockSf.excludeFromMinorCompaction()).thenReturn(false); when(mockSf.isReference()).thenReturn(false); // TODO come back to // this when selection takes this into account when(mockSf.getReader()).thenReturn(reader); String toString = MoreObjects.toStringHelper("MockStoreFile") .add("isReference", false) .add("fileSize", StringUtils.humanReadableInt(sizeInBytes)) .add("seqId", seqId) .add("path", stringPath).toString(); when(mockSf.toString()).thenReturn(toString); return mockSf; } }
@Test public void testWithReferences() throws Exception { StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create()); StripeCompactor sc = mock(StripeCompactor.class); HStoreFile ref = createFile(); when(ref.isReference()).thenReturn(true); StripeInformationProvider si = mock(StripeInformationProvider.class); Collection<HStoreFile> sfs = al(ref, createFile()); when(si.getStorefiles()).thenReturn(sfs); assertTrue(policy.needsCompactions(si, al())); StripeCompactionPolicy.StripeCompactionRequest scr = policy.selectCompaction(si, al(), false); // UnmodifiableCollection does not implement equals so we need to change it here to a // collection that implements it. assertEquals(si.getStorefiles(), new ArrayList<>(scr.getRequest().getFiles())); scr.execute(sc, NoLimitThroughputController.INSTANCE, null); verify(sc, only()).compact(eq(scr.getRequest()), anyInt(), anyLong(), aryEq(OPEN_KEY), aryEq(OPEN_KEY), aryEq(OPEN_KEY), aryEq(OPEN_KEY), any(), any()); }
.forEach(r -> r.getStores().get(0).getStorefiles().stream() .filter( s -> s.isReference() && !scanner.containsKey(r.getRegionInfo().getEncodedName())) .forEach(sf -> { StoreFileReader reader = ((HStoreFile) sf).getReader();
protected HStoreFile createMockStoreFile(final long sizeInBytes, final long seqId) { HStoreFile mockSf = mock(HStoreFile.class); StoreFileReader reader = mock(StoreFileReader.class); String stringPath = "/hbase/testTable/regionA/" + RandomStringUtils.random(FILENAME_LENGTH, 0, 0, true, true, null, random); Path path = new Path(stringPath); when(reader.getSequenceID()).thenReturn(seqId); when(reader.getTotalUncompressedBytes()).thenReturn(sizeInBytes); when(reader.length()).thenReturn(sizeInBytes); when(mockSf.getPath()).thenReturn(path); when(mockSf.excludeFromMinorCompaction()).thenReturn(false); when(mockSf.isReference()).thenReturn(false); // TODO come back to // this when selection takes this into account when(mockSf.getReader()).thenReturn(reader); String toString = MoreObjects.toStringHelper("MockStoreFile") .add("isReference", false) .add("fileSize", StringUtils.humanReadableInt(sizeInBytes)) .add("seqId", seqId) .add("path", stringPath).toString(); when(mockSf.toString()).thenReturn(toString); return mockSf; } }
@Test public void testWithReferences() throws Exception { StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create()); StripeCompactor sc = mock(StripeCompactor.class); HStoreFile ref = createFile(); when(ref.isReference()).thenReturn(true); StripeInformationProvider si = mock(StripeInformationProvider.class); Collection<HStoreFile> sfs = al(ref, createFile()); when(si.getStorefiles()).thenReturn(sfs); assertTrue(policy.needsCompactions(si, al())); StripeCompactionPolicy.StripeCompactionRequest scr = policy.selectCompaction(si, al(), false); // UnmodifiableCollection does not implement equals so we need to change it here to a // collection that implements it. assertEquals(si.getStorefiles(), new ArrayList<>(scr.getRequest().getFiles())); scr.execute(sc, NoLimitThroughputController.INSTANCE, null); verify(sc, only()).compact(eq(scr.getRequest()), anyInt(), anyLong(), aryEq(OPEN_KEY), aryEq(OPEN_KEY), aryEq(OPEN_KEY), aryEq(OPEN_KEY), any(), any()); }
.forEach(r -> r.getStores().get(0).getStorefiles().stream() .filter( s -> s.isReference() && !scanner.containsKey(r.getRegionInfo().getEncodedName())) .forEach(sf -> { StoreFileReader reader = ((HStoreFile) sf).getReader();