byte[] val = Bytes.toBytes("testval"); DefaultMemStore memstore = new DefaultMemStore(); memstore.add(new KeyValue(row, fam, qf1, val), null); memstore.add(new KeyValue(row, fam, qf2, val), null); memstore.add(new KeyValue(row, fam, qf3, val), null); MemStoreSnapshot snapshot = memstore.snapshot(); assertEquals(3, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam, qf4, val), null); memstore.add(new KeyValue(row, fam, qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount()); memstore.clearSnapshot(snapshot.getId());
/** * @param cell Find the row that comes after this one. If null, we return the * first. * @return Next row or null if none found. */ Cell getNextRow(final Cell cell) { return getLowest( getNextRow(cell, this.getActive().getCellSet()), getNextRow(cell, this.snapshot.getCellSet())); }
@Override /* * Scanners are ordered from 0 (oldest) to newest in increasing order. */ public List<KeyValueScanner> getScanners(long readPt) throws IOException { List<KeyValueScanner> list = new ArrayList<>(); addToScanners(getActive(), readPt, list); addToScanners(snapshot.getAllSegments(), readPt, list); return list; }
@Override public MemStoreSize getFlushableSize() { MemStoreSize mss = getSnapshotSize(); return mss.getDataSize() > 0? mss: getActive().getMemStoreSize(); }
EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); EnvironmentEdgeManager.injectEdge(edge); DefaultMemStore memstore = new DefaultMemStore(); long t = memstore.timeOfOldestEdit(); assertEquals(Long.MAX_VALUE, t); memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null); t = memstore.timeOfOldestEdit(); assertTrue(t == 1234); memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null); t = memstore.timeOfOldestEdit(); assertTrue(t == 1234); t = runSnapshot(memstore); kv1.setSequenceId(100); l.add(kv1); memstore.upsert(l, 1000, null); t = memstore.timeOfOldestEdit(); assertTrue(t == 1234); } finally {
@Test public void testMultipleVersionsSimple() throws Exception { DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR); byte [] row = Bytes.toBytes("testRow"); byte [] family = Bytes.toBytes("testFamily"); byte [] qf = Bytes.toBytes("testQualifier"); long [] stamps = {1,2,3}; byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"), Bytes.toBytes("value2")}; KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]); KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]); KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]); m.add(key0, null); m.add(key1, null); m.add(key2, null); assertTrue("Expected memstore to hold 3 values, actually has " + m.getActive().getCellsCount(), m.getActive().getCellsCount() == 3); }
public static void main(String [] args) throws IOException { MemStore ms = new DefaultMemStore(); long n1 = System.nanoTime(); addRows(25000, ms); System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000); System.out.println("foo"); for (int i = 0 ; i < 50 ; i++) doScan(ms, i); } }
runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion()); LOG.info("vmInputArguments=" + runtime.getInputArguments()); DefaultMemStore memstore1 = new DefaultMemStore(); for (int i = 0; i < count; i++) { memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memStoreSizing); memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memStoreSizing); memStoreSizing.getMemStoreSize().getHeapSize()); DefaultMemStore memstore2 = new DefaultMemStore(); memStoreSizing = new NonThreadSafeMemStoreSizing(); for (int i = 0; i < count; i++) { memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memStoreSizing);
/** * @param cell Find the row that comes after this one. If null, we return the * first. * @return Next row or null if none found. */ Cell getNextRow(final Cell cell) { return getLowest(getNextRow(cell, this.cellSet), getNextRow(cell, this.snapshot)); }
Cell closestToEmpty = ((DefaultMemStore) this.memstore).getNextRow(KeyValue.LOWESTKEY); assertTrue(CellComparatorImpl.COMPARATOR.compareRows(closestToEmpty, new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0); for (int i = 0; i < ROW_COUNT; i++) { Cell nr = ((DefaultMemStore) this.memstore).getNextRow(new KeyValue(Bytes.toBytes(i), System.currentTimeMillis())); if (i + 1 == ROW_COUNT) {
@Override protected List<Segment> getSegments() throws IOException { List<Segment> list = new ArrayList<>(2); list.add(getActive()); list.add(snapshot); return list; }
/** * Creates a snapshot of the current memstore. * Snapshot must be cleared by call to {@link #clearSnapshot(long)} */ @Override public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); if (!getActive().isEmpty()) { // Record the ImmutableSegment' heap overhead when initialing MemStoreSizing memstoreAccounting = new NonThreadSafeMemStoreSizing(); ImmutableSegment immutableSegment = SegmentFactory.instance(). createImmutableSegment(getActive(), memstoreAccounting); // regionServices can be null when testing if (regionServices != null) { regionServices.addMemStoreSize(memstoreAccounting.getDataSize(), memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(), memstoreAccounting.getCellsCount()); } this.snapshot = immutableSegment; resetActive(); } } return new MemStoreSnapshot(this.snapshotId, this.snapshot); }
/** * Write a delete * @param deleteCell * @return approximate size of the passed key and value. */ @Override public long delete(Cell deleteCell) { long s = 0; Cell toAdd = maybeCloneWithAllocator(deleteCell); s += heapSizeChange(toAdd, addToCellSet(toAdd)); timeRangeTracker.includeTimestamp(toAdd); this.size.addAndGet(s); return s; }
/** * Update or insert the specified KeyValues. * <p> * For each KeyValue, insert into MemStore. This will atomically upsert the * value for that row/family/qualifier. If a KeyValue did already exist, * it will then be removed. * <p> * Currently the memstoreTS is kept at 0 so as each insert happens, it will * be immediately visible. May want to change this so it is atomic across * all KeyValues. * <p> * This is called under row lock, so Get operations will still see updates * atomically. Scans will only see each KeyValue update as atomic. * * @param cells * @param readpoint readpoint below which we can safely remove duplicate KVs * @return change in memstore size */ @Override public long upsert(Iterable<Cell> cells, long readpoint) { long size = 0; for (Cell cell : cells) { size += upsert(cell, readpoint); } return size; }
/** * Internal version of add() that doesn't clone Cells with the * allocator, and doesn't take the lock. * * Callers should ensure they already have the read lock taken */ private long internalAdd(final Cell toAdd) { long s = heapSizeChange(toAdd, addToCellSet(toAdd)); timeRangeTracker.includeTimestamp(toAdd); this.size.addAndGet(s); return s; }
EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); EnvironmentEdgeManager.injectEdge(edge); DefaultMemStore memstore = new DefaultMemStore(); long t = memstore.timeOfOldestEdit(); assertEquals(Long.MAX_VALUE, t); memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null); t = memstore.timeOfOldestEdit(); assertTrue(t == 1234); memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null); t = memstore.timeOfOldestEdit(); assertTrue(t == 1234); t = runSnapshot(memstore); kv1.setSequenceId(100); l.add(kv1); memstore.upsert(l, 1000, null); t = memstore.timeOfOldestEdit(); assertTrue(t == 1234); } finally {
@Test public void testMultipleVersionsSimple() throws Exception { DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR); byte [] row = Bytes.toBytes("testRow"); byte [] family = Bytes.toBytes("testFamily"); byte [] qf = Bytes.toBytes("testQualifier"); long [] stamps = {1,2,3}; byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"), Bytes.toBytes("value2")}; KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]); KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]); KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]); m.add(key0, null); m.add(key1, null); m.add(key2, null); assertTrue("Expected memstore to hold 3 values, actually has " + m.getActive().getCellsCount(), m.getActive().getCellsCount() == 3); }
@Before public void setUp() throws Exception { internalSetUp(); // no pool this.chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); this.memstore = new DefaultMemStore(); }
runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion()); LOG.info("vmInputArguments=" + runtime.getInputArguments()); DefaultMemStore memstore1 = new DefaultMemStore(); for (int i = 0; i < count; i++) { size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty)); size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty)); DefaultMemStore memstore2 = new DefaultMemStore(); for (int i = 0; i < count; i++) { size += memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]));
Cell closestToEmpty = ((DefaultMemStore) this.memstore).getNextRow(KeyValue.LOWESTKEY); assertTrue(CellComparatorImpl.COMPARATOR.compareRows(closestToEmpty, new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0); for (int i = 0; i < ROW_COUNT; i++) { Cell nr = ((DefaultMemStore) this.memstore).getNextRow(new KeyValue(Bytes.toBytes(i), System.currentTimeMillis())); if (i + 1 == ROW_COUNT) {