/** * Computes the size of the store files for a single region. */ long getSizeOfStoreFile(TableName tn, StoreFileReference storeFileName) { String regionName = storeFileName.getRegionName(); return storeFileName.getFamilyToFilesMapping() .entries().stream() .collect(Collectors.summingLong((e) -> getSizeOfStoreFile(tn, regionName, e.getKey(), e.getValue()))); }
Collection<HbckInfo> ranges = regions.get(key); if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) { for (HbckInfo rng : ranges) { problemKey = key; overlapGroups.putAll(problemKey, ranges); for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) { handler.handleOverlapGroup(overlap); + "': overlap groups"); dumpOverlapProblems(overlapGroups); errors.print("There are " + overlapGroups.keySet().size() + " overlap groups with " + overlapGroups.size() + " overlapping regions");
void addFamilyStoreFile(String family, String storeFileName) { familyToFiles.put(family, storeFileName); }
private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey) throws IOException { // we parallelize overlap handler for the case we have lots of groups to fix. We can // safely assume each group is independent. List<WorkItemOverlapMerge> merges = new ArrayList<>(overlapGroups.size()); List<Future<Void>> rets; for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) { // merges.add(new WorkItemOverlapMerge(overlap, handler)); } try { rets = executor.invokeAll(merges); } catch (InterruptedException e) { LOG.error("Overlap merges were interrupted", e); return false; } for(int i=0; i<merges.size(); i++) { WorkItemOverlapMerge work = merges.get(i); Future<Void> f = rets.get(i); try { f.get(); } catch(ExecutionException e) { LOG.warn("Failed to merge overlap group" + work, e.getCause()); } catch (InterruptedException e) { LOG.error("Waiting for overlap merges was interrupted", e); return false; } } return true; }
/** * Generates a coverage multimap from split key to Regions that start with the * split key. * * @return coverage multimap */ public Multimap<byte[], R> calcCoverage() { // This needs to be sorted to force the use of the comparator on the values, // otherwise byte array comparison isn't used Multimap<byte[], R> regions = TreeMultimap.create(BYTES_COMPARATOR, rangeCmp); // march through all splits from the start points for (Entry<byte[], Collection<R>> start : starts.asMap().entrySet()) { byte[] key = start.getKey(); for (R r : start.getValue()) { regions.put(key, r); for (byte[] coveredSplit : splits.subSet(r.getStartKey(), specialEndKey(r))) { regions.put(coveredSplit, r); } } } return regions; }
public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) { // we display this way because the last end key should be displayed as // well. for (byte[] k : regions.keySet()) { errors.print(Bytes.toStringBinary(k) + ":"); for (HbckInfo r : regions.get(k)) { errors.print("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]"); } errors.print("----"); } }
@Test public void testSnapshotsFromTables() throws Exception { TableName tn1 = helper.createTableWithRegions(1); TableName tn2 = helper.createTableWithRegions(1); TableName tn3 = helper.createTableWithRegions(1); // Set a space quota on table 1 and 2 (but not 3) admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); // Create snapshots on each table (we didn't write any data, so just skipflush) admin.snapshot(new SnapshotDescription(tn1 + "snapshot", tn1, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn2 + "snapshot", tn2, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn3 + "snapshot", tn3, SnapshotType.SKIPFLUSH)); Multimap<TableName,String> mapping = testChore.getSnapshotsToComputeSize(); assertEquals(2, mapping.size()); assertEquals(1, mapping.get(tn1).size()); assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next()); assertEquals(1, mapping.get(tn2).size()); assertEquals(tn2 + "snapshot", mapping.get(tn2).iterator().next()); admin.snapshot(new SnapshotDescription(tn2 + "snapshot1", tn2, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn3 + "snapshot1", tn3, SnapshotType.SKIPFLUSH)); mapping = testChore.getSnapshotsToComputeSize(); assertEquals(3, mapping.size()); assertEquals(1, mapping.get(tn1).size()); assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next()); assertEquals(2, mapping.get(tn2).size()); assertEquals( new HashSet<String>(Arrays.asList(tn2 + "snapshot", tn2 + "snapshot1")), mapping.get(tn2)); }
/** * This dumps data in a visually reasonable way for visual debugging * * @param splits * @param regions */ void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) { // we display this way because the last end key should be displayed as well. StringBuilder sb = new StringBuilder(); for (byte[] k : splits) { sb.setLength(0); // clear out existing buffer, if any. sb.append(Bytes.toStringBinary(k) + ":\t"); for (HbckInfo r : regions.get(k)) { sb.append("[ "+ r.toString() + ", " + Bytes.toStringBinary(r.getEndKey())+ "]\t"); } errors.print(sb.toString()); } } }
@Override public Set<K> keySet() { return fromMultimap.keySet(); }
public int getNumRegions() { return sc.getStarts().size() + backwards.size(); }
for (Entry<ByteBuffer, ? extends Collection<LoadQueueItem>> e : regionGroups.asMap() .entrySet()) { byte[] first = e.getKey().array();
/** * Returns a newly-created immutable multimap. */ public ImmutableMultimap<K, V> build() { if (valueComparator != null) { for (Collection<V> values : builderMultimap.asMap().values()) { List<V> list = (List<V>) values; Collections.sort(list, valueComparator); } } if (keyComparator != null) { Multimap<K, V> sortedCopy = MultimapBuilder.linkedHashKeys().arrayListValues().<K, V>build(); List<Map.Entry<K, Collection<V>>> entries = Ordering.from(keyComparator) .<K>onKeys() .immutableSortedCopy(builderMultimap.asMap().entrySet()); for (Map.Entry<K, Collection<V>> entry : entries) { sortedCopy.putAll(entry.getKey(), entry.getValue()); } builderMultimap = sortedCopy; } return copyOf(builderMultimap); } }
private static <K, V> ImmutableSetMultimap<K, V> copyOf( Multimap<? extends K, ? extends V> multimap, Comparator<? super V> valueComparator) { checkNotNull(multimap); // eager for GWT if (multimap.isEmpty() && valueComparator == null) { return of(); } if (multimap instanceof ImmutableSetMultimap) { @SuppressWarnings("unchecked") // safe since multimap is not writable ImmutableSetMultimap<K, V> kvMultimap = (ImmutableSetMultimap<K, V>) multimap; if (!kvMultimap.isPartialView()) { return kvMultimap; } } ImmutableMap.Builder<K, ImmutableSet<V>> builder = new ImmutableMap.Builder<K, ImmutableSet<V>>(multimap.asMap().size()); int size = 0; for (Entry<? extends K, ? extends Collection<? extends V>> entry : multimap.asMap().entrySet()) { K key = entry.getKey(); Collection<? extends V> values = entry.getValue(); ImmutableSet<V> set = valueSet(valueComparator, values); if (!set.isEmpty()) { builder.put(key, set); size += set.size(); } } return new ImmutableSetMultimap<K, V>(builder.build(), size, valueComparator); }
snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss1", 1024L)); snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss2", 4096L)); testChore.persistSnapshotSizes(table, snapshotsWithSizes); assertEquals(2, count(table)); }); snapshotsWithSizes.clear(); verify(table, () -> { snapshotsWithSizes.put(TableName.valueOf("t1"), new SnapshotWithSize("ss1", 1024L)); snapshotsWithSizes.put(TableName.valueOf("t2"), new SnapshotWithSize("ss2", 4096L)); snapshotsWithSizes.put(TableName.valueOf("t3"), new SnapshotWithSize("ss3", 8192L)); testChore.persistSnapshotSizes(table, snapshotsWithSizes); assertEquals(3, count(table));
@Override public void clear() { multimap.clear(); } }
if (!regionReference.getFamilyToFilesMapping().isEmpty()) { references.add(regionReference);
@Override public boolean putAll(Multimap<? extends K, ? extends V> multimap) { synchronized (mutex) { return delegate().putAll(multimap); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof StoreFileReference)) { return false; } StoreFileReference other = (StoreFileReference) o; return regionName.equals(other.regionName) && familyToFiles.equals(other.familyToFiles); }