@Override public long getEarliestMemStoreSeqNum(byte[] encodedRegionName, byte[] familyName) { // This method is used by tests and for figuring if we should flush or not because our // sequenceids are too old. It is also used reporting the master our oldest sequenceid for use // figuring what edits can be skipped during log recovery. getEarliestMemStoreSequenceId // from this.sequenceIdAccounting is looking first in flushingOldestStoreSequenceIds, the // currently flushing sequence ids, and if anything found there, it is returning these. This is // the right thing to do for the reporting oldest sequenceids to master; we won't skip edits if // we crash during the flush. For figuring what to flush, we might get requeued if our sequence // id is old even though we are currently flushing. This may mean we do too much flushing. return this.sequenceIdAccounting.getLowestSequenceId(encodedRegionName, familyName); }
/** * Returns the lowest unflushed sequence id for the region. * @param encodedRegionName * @return Lowest outstanding unflushed sequenceid for <code>encodedRegionName</code>. Will * return {@link HConstants#NO_SEQNUM} when none. */ long getLowestSequenceId(final byte[] encodedRegionName) { synchronized (this.tieLock) { Map<?, Long> m = this.flushingSequenceIds.get(encodedRegionName); long flushingLowest = m != null ? getLowestSequenceId(m) : Long.MAX_VALUE; m = this.lowestUnflushedSequenceIds.get(encodedRegionName); long unflushedLowest = m != null ? getLowestSequenceId(m) : HConstants.NO_SEQNUM; return Math.min(flushingLowest, unflushedLowest); } }
/** * @param src * @return New Map that has same keys as <code>src</code> but instead of a Map for a value, it * instead has found the smallest sequence id and it returns that as the value instead. */ private <T extends Map<?, Long>> Map<byte[], Long> flattenToLowestSequenceId(Map<byte[], T> src) { if (src == null || src.isEmpty()) { return null; } Map<byte[], Long> tgt = new HashMap<>(); for (Map.Entry<byte[], T> entry : src.entrySet()) { long lowestSeqId = getLowestSequenceId(entry.getValue()); if (lowestSeqId != HConstants.NO_SEQNUM) { tgt.put(entry.getKey(), lowestSeqId); } } return tgt; }
/** * Iterates over the given Map and compares sequence ids with corresponding entries in * {@link #lowestUnflushedSequenceIds}. If a region in * {@link #lowestUnflushedSequenceIds} has a sequence id less than that passed in * <code>sequenceids</code> then return it. * @param sequenceids Sequenceids keyed by encoded region name. * @return regions found in this instance with sequence ids less than those passed in. */ byte[][] findLower(Map<byte[], Long> sequenceids) { List<byte[]> toFlush = null; // Keeping the old behavior of iterating unflushedSeqNums under oldestSeqNumsLock. synchronized (tieLock) { for (Map.Entry<byte[], Long> e : sequenceids.entrySet()) { Map<ImmutableByteArray, Long> m = this.lowestUnflushedSequenceIds.get(e.getKey()); if (m == null) { continue; } // The lowest sequence id outstanding for this region. long lowest = getLowestSequenceId(m); if (lowest != HConstants.NO_SEQNUM && lowest <= e.getValue()) { if (toFlush == null) { toFlush = new ArrayList<>(); } toFlush.add(e.getKey()); } } } return toFlush == null ? null : toFlush.toArray(new byte[0][]); } }
@Test public void testFindLower() { SequenceIdAccounting sida = new SequenceIdAccounting(); sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME); Map<byte[], Long> m = new HashMap<>(); m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM); long sequenceid = 1; sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); assertTrue(sida.findLower(m) == null); m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME)); assertTrue(sida.findLower(m).length == 1); m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME) - 1); assertTrue(sida.findLower(m) == null); } }
assertEquals(1, regions.size()); assertArrayEquals(ENCODED_REGION_NAME, regions.get(0)); long lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME); assertEquals("Lowest should be first sequence id inserted", 1, lowest); m.put(ENCODED_REGION_NAME, lowest); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME); m.put(ENCODED_REGION_NAME, lowest); assertFalse(sida.areAllLower(m, null)); sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES); assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME)); sida.completeCacheFlush(ENCODED_REGION_NAME); assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME));
@Override public long getEarliestMemstoreSeqNum(byte[] encodedRegionName, byte[] familyName) { // This method is used by tests and for figuring if we should flush or not because our // sequenceids are too old. It is also used reporting the master our oldest sequenceid for use // figuring what edits can be skipped during log recovery. getEarliestMemStoreSequenceId // from this.sequenceIdAccounting is looking first in flushingOldestStoreSequenceIds, the // currently flushing sequence ids, and if anything found there, it is returning these. This is // the right thing to do for the reporting oldest sequenceids to master; we won't skip edits if // we crash during the flush. For figuring what to flush, we might get requeued if our sequence // id is old even though we are currently flushing. This may mean we do too much flushing. return this.sequenceIdAccounting.getLowestSequenceId(encodedRegionName, familyName); }
/** * Returns the lowest unflushed sequence id for the region. * @param encodedRegionName * @return Lowest outstanding unflushed sequenceid for <code>encodedRegionName</code>. Will * return {@link HConstants#NO_SEQNUM} when none. */ long getLowestSequenceId(final byte [] encodedRegionName) { synchronized (this.tieLock) { Map<byte[], Long> m = this.flushingSequenceIds.get(encodedRegionName); long flushingLowest = m != null? getLowestSequenceId(m): Long.MAX_VALUE; m = this.lowestUnflushedSequenceIds.get(encodedRegionName); long unflushedLowest = m != null? getLowestSequenceId(m): HConstants.NO_SEQNUM; return Math.min(flushingLowest, unflushedLowest); } }
/** * @param src * @return New Map that has same keys as <code>src</code> but instead of a Map for a value, it * instead has found the smallest sequence id and it returns that as the value instead. */ private <T extends Map<byte[], Long>> Map<byte[], Long> flattenToLowestSequenceId( Map<byte[], T> src) { if (src == null || src.isEmpty()) return null; Map<byte[], Long> tgt = Maps.newHashMap(); for (Map.Entry<byte[], T> entry: src.entrySet()) { long lowestSeqId = getLowestSequenceId(entry.getValue()); if (lowestSeqId != HConstants.NO_SEQNUM) { tgt.put(entry.getKey(), lowestSeqId); } } return tgt; }
/** * Iterates over the given Map and compares sequence ids with corresponding * entries in {@link #oldestUnflushedRegionSequenceIds}. If a region in * {@link #oldestUnflushedRegionSequenceIds} has a sequence id less than that passed * in <code>sequenceids</code> then return it. * @param sequenceids Sequenceids keyed by encoded region name. * @return regions found in this instance with sequence ids less than those passed in. */ byte[][] findLower(Map<byte[], Long> sequenceids) { List<byte[]> toFlush = null; // Keeping the old behavior of iterating unflushedSeqNums under oldestSeqNumsLock. synchronized (tieLock) { for (Map.Entry<byte[], Long> e: sequenceids.entrySet()) { Map<byte[], Long> m = this.lowestUnflushedSequenceIds.get(e.getKey()); if (m == null) continue; // The lowest sequence id outstanding for this region. long lowest = getLowestSequenceId(m); if (lowest != HConstants.NO_SEQNUM && lowest <= e.getValue()) { if (toFlush == null) toFlush = new ArrayList<byte[]>(); toFlush.add(e.getKey()); } } } return toFlush == null? null: toFlush.toArray(new byte[][] { HConstants.EMPTY_BYTE_ARRAY }); } }
@Test public void testFindLower() { SequenceIdAccounting sida = new SequenceIdAccounting(); sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME); Map<byte[], Long> m = new HashMap<>(); m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM); long sequenceid = 1; sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); assertTrue(sida.findLower(m) == null); m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME)); assertTrue(sida.findLower(m).length == 1); m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME) - 1); assertTrue(sida.findLower(m) == null); } }
m.put(ENCODED_REGION_NAME, sequenceid); assertFalse(sida.areAllLower(m)); long lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME); assertEquals("Lowest should be first sequence id inserted", 1, lowest); m.put(ENCODED_REGION_NAME, lowest); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME); m.put(ENCODED_REGION_NAME, lowest); assertFalse(sida.areAllLower(m)); sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES); assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME)); sida.completeCacheFlush(ENCODED_REGION_NAME); assertEquals(HConstants.NO_SEQNUM, sida.getLowestSequenceId(ENCODED_REGION_NAME));