@VisibleForTesting void flushInMemory() { MutableSegment currActive = getActive(); if(currActive.setInMemoryFlushed()) { flushInMemory(currActive); } inMemoryCompaction(); }
/** * Check whether anything need to be done based on the current active set size. * The method is invoked upon every addition to the active set. * For CompactingMemStore, flush the active set to the read-only memory if it's * size is above threshold * @param currActive intended segment to update * @param cellToAdd cell to be added to the segment * @param memstoreSizing object to accumulate changed size * @return true if the cell can be added to the */ private boolean checkAndAddToActiveSize(MutableSegment currActive, Cell cellToAdd, MemStoreSizing memstoreSizing) { if (shouldFlushInMemory(currActive, cellToAdd, memstoreSizing)) { if (currActive.setInMemoryFlushed()) { flushInMemory(currActive); if (setInMemoryCompactionFlag()) { // The thread is dispatched to do in-memory compaction in the background InMemoryCompactionRunnable runnable = new InMemoryCompactionRunnable(); if (LOG.isTraceEnabled()) { LOG.trace("Dispatching the MemStore in-memory flush for store " + store .getColumnFamilyName()); } getPool().execute(runnable); } } return false; } return true; }
@Test public void testCountOfCellsAfterFlatteningByScan() throws IOException { String[] keys1 = { "A", "B", "C" }; // A, B, C addRowsByKeysWith50Cols(memstore, keys1); // this should only flatten as there are no duplicates ((CompactingMemStore) memstore).flushInMemory(); while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10); } List<KeyValueScanner> scanners = memstore.getScanners(Long.MAX_VALUE); // seek int count = 0; for(int i = 0; i < scanners.size(); i++) { scanners.get(i).seek(KeyValue.LOWESTKEY); while (scanners.get(i).next() != null) { count++; } } assertEquals("the count should be ", 150, count); for(int i = 0; i < scanners.size(); i++) { scanners.get(i).close(); } }
region.flush(true); // enforce flush of everything TO DISK while there are still ongoing puts ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).flushInMemory(); ((CompactingMemStore) ((HStore)region.getStore(FAMILY3)).memstore).flushInMemory(); while (((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore) .isMemStoreFlushingInMemory()) {
m -> m.getTimeRangeTracker().getMax()).max().getAsLong()); ((CompactingMemStore) memstore).flushInMemory(); m -> m.getTimeRangeTracker().getMax()).max().getAsLong()); ((CompactingMemStore) memstore).flushInMemory(); // trigger the merge
.flushInMemory(); while (((CompactingMemStore) ((HStore) stressedRegion.getStore(FAMILY1)).memstore) .isMemStoreFlushingInMemory()) { ((CompactingMemStore) ((HStore) stressedRegion.getStore(FAMILY2)).memstore).flushInMemory(); while (((CompactingMemStore) ((HStore) stressedRegion.getStore(FAMILY2)).memstore) .isMemStoreFlushingInMemory()) { ((CompactingMemStore) ((HStore) stressedRegion.getStore(FAMILY3)).memstore).flushInMemory(); while (((CompactingMemStore) ((HStore) stressedRegion.getStore(FAMILY3)).memstore) .isMemStoreFlushingInMemory()) {
@Override @Test public void testGet_memstoreAndSnapShot() throws IOException { byte[] row = Bytes.toBytes("testrow"); byte[] fam = Bytes.toBytes("testfamily"); byte[] qf1 = Bytes.toBytes("testqualifier1"); byte[] qf2 = Bytes.toBytes("testqualifier2"); byte[] qf3 = Bytes.toBytes("testqualifier3"); byte[] qf4 = Bytes.toBytes("testqualifier4"); byte[] qf5 = Bytes.toBytes("testqualifier5"); byte[] val = Bytes.toBytes("testval"); //Setting up memstore memstore.add(new KeyValue(row, fam, qf1, val), null); memstore.add(new KeyValue(row, fam, qf2, val), null); memstore.add(new KeyValue(row, fam, qf3, val), null); //Pushing to pipeline ((CompactingMemStore)memstore).flushInMemory(); assertEquals(0, memstore.getSnapshot().getCellsCount()); //Creating a snapshot memstore.snapshot(); assertEquals(3, memstore.getSnapshot().getCellsCount()); //Adding value to "new" memstore assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam, qf4, val), null); memstore.add(new KeyValue(row, fam, qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount()); }
assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and flatten assertEquals(0, memstore.getSnapshot().getCellsCount()); long oneCellOnCCMHeapSize =
@Test public void testCountOfCellsAfterFlatteningByIterator() throws IOException { String[] keys1 = { "A", "B", "C" }; // A, B, C addRowsByKeysWith50Cols(memstore, keys1); // this should only flatten as there are no duplicates ((CompactingMemStore) memstore).flushInMemory(); while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10); } // Just doing the cnt operation here MemStoreSegmentsIterator itr = new MemStoreMergerSegmentsIterator( ((CompactingMemStore) memstore).getImmutableSegments().getStoreSegments(), CellComparatorImpl.COMPARATOR, 10); int cnt = 0; try { while (itr.next() != null) { cnt++; } } finally { itr.close(); } assertEquals("the count should be ", 150, cnt); }
assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and flatten while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10);
((CompactingMemStore)memstore).flushInMemory(); ((CompactingMemStore)memstore).flushInMemory(); ((CompactingMemStore)memstore).flushInMemory();
addRowsByKeysDataSize(memstore, keys1); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline should not compact ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without flattening assertEquals(0, memstore.getSnapshot().getCellsCount()); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10);
assertEquals(totalHeapSize, memstore.heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline - flatten assertEquals(3, ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells()); assertEquals(1.0, ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline without compaction assertEquals(4, ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells()); assertEquals(1.0, ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline without compaction assertEquals(8, ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells()); assertEquals((4.0 / 8.0), ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact int numCells = ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells(); assertTrue(4 == numCells || 11 == numCells);
assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact int counter = 0; for ( Segment s : memstore.getSegments()) { ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemStoreSize());
assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact assertEquals(0, memstore.getSnapshot().getCellsCount());
long totalHeapSize = MutableSegment.DEEP_OVERHEAD + 4 * oneCellOnCSLMHeapSize; assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline without compaction assertEquals(0, memstore.getSnapshot().getCellsCount()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact assertEquals(0, memstore.getSnapshot().getCellsCount());
assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact int counter = 0; // COMPACT 4->3 for ( Segment s : memstore.getSegments()) { assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact assertEquals(0, memstore.getSnapshot().getCellsCount());// COMPACT 6->4 counter = 0;
((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without compaction totalHeapSize2 = totalHeapSize2 + CSLMImmutableSegment.DEEP_OVERHEAD_CSLM; assertEquals(0, memstore.getSnapshot().getCellsCount()); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10);
((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact assertEquals(0, memstore.getSnapshot().getCellsCount());