Refine search
@Test public void testScanMetrics() throws Exception { Pair<List<Result>, ScanMetrics> pair = method.scan(new Scan().setScanMetricsEnabled(true)); List<Result> results = pair.getFirst(); assertEquals(3, results.size()); long bytes = results.stream().flatMap(r -> Arrays.asList(r.rawCells()).stream()) .mapToLong(c -> PrivateCellUtil.estimatedSerializedSizeOf(c)).sum(); ScanMetrics scanMetrics = pair.getSecond(); assertEquals(NUM_REGIONS, scanMetrics.countOfRegions.get()); assertEquals(bytes, scanMetrics.countOfBytesInResults.get()); assertEquals(NUM_REGIONS, scanMetrics.countOfRPCcalls.get()); // also assert a server side metric to ensure that we have published them into the client side // metrics. assertEquals(3, scanMetrics.countOfRowsScanned.get()); } }
static void verifyResult(Result result, List<Cell> expKvList, boolean toLog, String msg) { LOG.info(msg); LOG.info("Expected count: " + expKvList.size()); LOG.info("Actual count: " + result.size()); if (expKvList.isEmpty()) return; int i = 0; for (Cell kv : result.rawCells()) { if (i >= expKvList.size()) { break; // we will check the size later } Cell kvExp = expKvList.get(i++); if (toLog) { LOG.info("get kv is: " + kv.toString()); LOG.info("exp kv is: " + kvExp.toString()); } assertTrue("Not equal", kvExp.equals(kv)); } assertEquals(expKvList.size(), result.size()); }
/** * Verifies that result contains all the key values within expKvList. Fails the test otherwise * @param result * @param expKvList * @param msg */ static void verifyResult(Result result, List<Cell> expKvList, String msg) { if (LOG.isInfoEnabled()) { LOG.info(msg); LOG.info("Expected count: " + expKvList.size()); LOG.info("Actual count: " + result.size()); } if (expKvList.isEmpty()) return; int i = 0; for (Cell kv : result.rawCells()) { if (i >= expKvList.size()) { break; // we will check the size later } Cell kvExp = expKvList.get(i++); assertTrue("Not equal. get kv: " + kv.toString() + " exp kv: " + kvExp.toString(), kvExp.equals(kv)); } assertEquals(expKvList.size(), result.size()); }
private Cell[] getData(String family, String row, List<String> columns, int expBlocksRowCol, int expBlocksRow, int expBlocksNone) throws IOException { int[] expBlocks = new int[] { expBlocksRowCol, expBlocksRow, expBlocksNone }; Cell[] kvs = null; for (int i = 0; i < BLOOM_TYPE.length; i++) { BloomType bloomType = BLOOM_TYPE[i]; byte[] cf = Bytes.toBytes(family + "_" + bloomType); long blocksStart = getBlkAccessCount(cf); Get get = new Get(Bytes.toBytes(row)); for (String column : columns) { get.addColumn(cf, Bytes.toBytes(column)); } kvs = region.get(get).rawCells(); long blocksEnd = getBlkAccessCount(cf); if (expBlocks[i] != -1) { assertEquals("Blocks Read Check for Bloom: " + bloomType, expBlocks[i], blocksEnd - blocksStart); } System.out.println("Blocks Read for Bloom: " + bloomType + " = " + (blocksEnd - blocksStart) + "Expected = " + expBlocks[i]); } return kvs; }
private void testPreBatchMutate(TableName tableName, Runnable rn)throws Exception { HTableDescriptor desc = new HTableDescriptor(tableName); desc.addCoprocessor(WaitingForScanObserver.class.getName()); desc.addFamily(new HColumnDescriptor(FAMILY)); TEST_UTIL.getAdmin().createTable(desc); ExecutorService service = Executors.newFixedThreadPool(2); service.execute(rn); final List<Cell> cells = new ArrayList<>(); service.execute(() -> { try { // waiting for update. TimeUnit.SECONDS.sleep(3); Table t = TEST_UTIL.getConnection().getTable(tableName); Scan scan = new Scan(); try (ResultScanner scanner = t.getScanner(scan)) { for (Result r : scanner) { cells.addAll(Arrays.asList(r.rawCells())); } } } catch (IOException | InterruptedException ex) { throw new RuntimeException(ex); } }); service.shutdown(); service.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); assertEquals("The write is blocking by RegionObserver#postBatchMutate" + ", so the data is invisible to reader", 0, cells.size()); TEST_UTIL.deleteTable(tableName); }
private static void assertOnlyLatest(final Table incommon, final long currentTime) throws IOException { Get get = null; get = new Get(ROW); get.addColumn(FAMILY_NAME, QUALIFIER_NAME); get.setMaxVersions(3); Result result = incommon.get(get); Assert.assertEquals(1, result.size()); long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0])); Assert.assertEquals(time, currentTime); }
@Test public void testCombine3() throws IOException { Result result1 = Result.create(Arrays.asList(createCell(1, CQ1)), null, false, true); Result result2 = Result.create(Arrays.asList(createCell(1, CQ2)), null, false, true); Result nextResult1 = Result.create(Arrays.asList(createCell(2, CQ1)), null, false, false); Result nextToNextResult1 = Result.create(Arrays.asList(createCell(3, CQ1)), null, false, true); assertEquals(0, resultCache.addAndGet(new Result[] { result1 }, false).length); assertEquals(0, resultCache.addAndGet(new Result[] { result2 }, false).length); Result[] results = resultCache.addAndGet(new Result[] { nextResult1, nextToNextResult1 }, false); assertEquals(2, results.length); assertEquals(1, Bytes.toInt(results[0].getRow())); assertEquals(2, results[0].rawCells().length); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ1))); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ2))); assertEquals(2, Bytes.toInt(results[1].getRow())); assertEquals(1, results[1].rawCells().length); assertEquals(2, Bytes.toInt(results[1].getValue(CF, CQ1))); results = resultCache.addAndGet(new Result[0], false); assertEquals(1, results.length); assertEquals(3, Bytes.toInt(results[0].getRow())); assertEquals(1, results[0].rawCells().length); assertEquals(3, Bytes.toInt(results[0].getValue(CF, CQ1))); }
@Test public void testCombine4() throws IOException { Result result1 = Result.create(Arrays.asList(createCell(1, CQ1)), null, false, true); Result result2 = Result.create(Arrays.asList(createCell(1, CQ2)), null, false, false); Result nextResult1 = Result.create(Arrays.asList(createCell(2, CQ1)), null, false, true); Result nextResult2 = Result.create(Arrays.asList(createCell(2, CQ2)), null, false, false); assertEquals(0, resultCache.addAndGet(new Result[] { result1 }, false).length); Result[] results = resultCache.addAndGet(new Result[] { result2, nextResult1 }, false); assertEquals(1, results.length); assertEquals(1, Bytes.toInt(results[0].getRow())); assertEquals(2, results[0].rawCells().length); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ1))); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ2))); results = resultCache.addAndGet(new Result[] { nextResult2 }, false); assertEquals(1, results.length); assertEquals(2, Bytes.toInt(results[0].getRow())); assertEquals(2, results[0].rawCells().length); assertEquals(2, Bytes.toInt(results[0].getValue(CF, CQ1))); assertEquals(2, Bytes.toInt(results[0].getValue(CF, CQ2))); } }
@Test public void testMayHaveMoreCellsInRowReturnsTrueAndSetBatch() throws IOException { Table table = createTestTable(TableName.valueOf(name.getMethodName()), ROWS, FAMILIES, QUALIFIERS, VALUE); Scan scan = new Scan(); scan.setBatch(1); scan.setFilter(new FirstKeyOnlyFilter()); ResultScanner scanner = table.getScanner(scan); Result result; while ((result = scanner.next()) != null) { assertTrue(result.rawCells() != null); assertEquals(1, result.rawCells().length); } }
@Test public void testCombine2() throws IOException { Result result1 = Result.create(Arrays.asList(createCell(1, CQ1)), null, false, true); Result result2 = Result.create(Arrays.asList(createCell(1, CQ2)), null, false, true); Result result3 = Result.create(Arrays.asList(createCell(1, CQ3)), null, false, true); Result nextResult1 = Result.create(Arrays.asList(createCell(2, CQ1)), null, false, true); Result nextToNextResult1 = Result.create(Arrays.asList(createCell(3, CQ2)), null, false, false); assertEquals(0, resultCache.addAndGet(new Result[] { result1 }, false).length); assertEquals(0, resultCache.addAndGet(new Result[] { result2 }, false).length); assertEquals(0, resultCache.addAndGet(new Result[] { result3 }, false).length); Result[] results = resultCache.addAndGet(new Result[] { nextResult1 }, false); assertEquals(1, results.length); assertEquals(1, Bytes.toInt(results[0].getRow())); assertEquals(3, results[0].rawCells().length); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ1))); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ2))); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ3))); results = resultCache.addAndGet(new Result[] { nextToNextResult1 }, false); assertEquals(2, results.length); assertEquals(2, Bytes.toInt(results[0].getRow())); assertEquals(1, results[0].rawCells().length); assertEquals(2, Bytes.toInt(results[0].getValue(CF, CQ1))); assertEquals(3, Bytes.toInt(results[1].getRow())); assertEquals(1, results[1].rawCells().length); assertEquals(3, Bytes.toInt(results[1].getValue(CF, CQ2))); }
private void assertICV(byte [] row, byte [] familiy, byte[] qualifier, long amount, boolean fast) throws IOException { // run a get and see? Get get = new Get(row); if (fast) get.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED); get.addColumn(familiy, qualifier); Result result = region.get(get); assertEquals(1, result.size()); Cell kv = result.rawCells()[0]; long r = Bytes.toLong(CellUtil.cloneValue(kv)); assertEquals(amount, r); }
@Test public void testAppendWithoutWAL() throws Exception { List<Result> resultsWithWal = doAppend(true); List<Result> resultsWithoutWal = doAppend(false); assertEquals(resultsWithWal.size(), resultsWithoutWal.size()); for (int i = 0; i != resultsWithWal.size(); ++i) { Result resultWithWal = resultsWithWal.get(i); Result resultWithoutWal = resultsWithoutWal.get(i); assertEquals(resultWithWal.rawCells().length, resultWithoutWal.rawCells().length); for (int j = 0; j != resultWithWal.rawCells().length; ++j) { Cell cellWithWal = resultWithWal.rawCells()[j]; Cell cellWithoutWal = resultWithoutWal.rawCells()[j]; assertTrue(Bytes.equals(CellUtil.cloneRow(cellWithWal), CellUtil.cloneRow(cellWithoutWal))); assertTrue(Bytes.equals(CellUtil.cloneFamily(cellWithWal), CellUtil.cloneFamily(cellWithoutWal))); assertTrue(Bytes.equals(CellUtil.cloneQualifier(cellWithWal), CellUtil.cloneQualifier(cellWithoutWal))); assertTrue(Bytes.equals(CellUtil.cloneValue(cellWithWal), CellUtil.cloneValue(cellWithoutWal))); } } }
private static void assertGet(final HRegion region, byte [] row, byte [] familiy, byte[] qualifier, byte[] value) throws IOException { // run a get and see if the value matches Get get = new Get(row); get.addColumn(familiy, qualifier); Result result = region.get(get); assertEquals(1, result.size()); Cell kv = result.rawCells()[0]; byte[] r = CellUtil.cloneValue(kv); assertTrue(Bytes.compareTo(r, value) == 0); }
@Test public void testRowAtomic() throws IOException, InterruptedException { byte[] row = Bytes.toBytes("row"); put(row, CQ1, Bytes.toBytes(1)); put(row, CQ2, Bytes.toBytes(2)); try (Table table = CONN.getTable(tableName); ResultScanner scanner = table.getScanner(new Scan().setBatch(1).setCaching(1))) { Result result = scanner.next(); assertEquals(1, result.rawCells().length); assertEquals(1, Bytes.toInt(result.getValue(CF, CQ1))); move(); put(row, CQ3, Bytes.toBytes(3)); result = scanner.next(); assertEquals(1, result.rawCells().length); assertEquals(2, Bytes.toInt(result.getValue(CF, CQ2))); assertNull(scanner.next()); } }
@Test public void testPutMasked2() throws IOException { try (Table t = createTable()) { t.put(new Put(ROW).addColumn(FAMILY, col1, 1000001, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000002, value)); t.delete(new Delete(ROW).addColumn(FAMILY, col1, 1000003)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000003, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000004, value)); Result r = t.get(new Get(ROW).setMaxVersions(3)); assertEquals(3, r.size()); assertEquals(1000004, r.rawCells()[0].getTimestamp()); assertEquals(1000003, r.rawCells()[1].getTimestamp()); assertEquals(1000002, r.rawCells()[2].getTimestamp()); TEST_UTIL.getAdmin().flush(t.getName()); r = t.get(new Get(ROW).setMaxVersions(3)); assertEquals(3, r.size()); assertEquals(1000004, r.rawCells()[0].getTimestamp()); assertEquals(1000003, r.rawCells()[1].getTimestamp()); assertEquals(1000002, r.rawCells()[2].getTimestamp()); } }
@Test public void testCombine1() throws IOException { Result previousResult = Result.create(Arrays.asList(createCell(0, CQ1)), null, false, true); Result result1 = Result.create(Arrays.asList(createCell(1, CQ1)), null, false, true); Result result2 = Result.create(Arrays.asList(createCell(1, CQ2)), null, false, true); Result result3 = Result.create(Arrays.asList(createCell(1, CQ3)), null, false, true); Result[] results = resultCache.addAndGet(new Result[] { previousResult, result1 }, false); assertEquals(1, results.length); assertSame(previousResult, results[0]); assertEquals(0, resultCache.addAndGet(new Result[] { result2 }, false).length); assertEquals(0, resultCache.addAndGet(new Result[] { result3 }, false).length); assertEquals(0, resultCache.addAndGet(new Result[0], true).length); results = resultCache.addAndGet(new Result[0], false); assertEquals(1, results.length); assertEquals(1, Bytes.toInt(results[0].getRow())); assertEquals(3, results[0].rawCells().length); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ1))); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ2))); assertEquals(1, Bytes.toInt(results[0].getValue(CF, CQ3))); }
@Test public void testSameTs() throws IOException { try (Table t = createTable()) { t.put(new Put(ROW).addColumn(FAMILY, col1, 1000001, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000002, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000003, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000003, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000003, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000004, value)); Result r = t.get(new Get(ROW).setMaxVersions(3)); assertEquals(3, r.size()); assertEquals(1000004, r.rawCells()[0].getTimestamp()); assertEquals(1000003, r.rawCells()[1].getTimestamp()); assertEquals(1000002, r.rawCells()[2].getTimestamp()); TEST_UTIL.getAdmin().flush(t.getName()); r = t.get(new Get(ROW).setMaxVersions(3)); assertEquals(3, r.size()); assertEquals(1000004, r.rawCells()[0].getTimestamp()); assertEquals(1000003, r.rawCells()[1].getTimestamp()); assertEquals(1000002, r.rawCells()[2].getTimestamp()); } }
@Test public void testIncrementWithCustomTimestamp() throws IOException { TableName TABLENAME = TableName.valueOf(name.getMethodName()); Table table = TEST_UTIL.createTable(TABLENAME, FAMILY); long timestamp = 999; Increment increment = new Increment(ROW); increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L))); Result r = table.increment(increment); assertEquals(1, r.size()); assertEquals(timestamp, r.rawCells()[0].getTimestamp()); r = table.get(new Get(ROW)); assertEquals(1, r.size()); assertEquals(timestamp, r.rawCells()[0].getTimestamp()); r = table.increment(increment); assertEquals(1, r.size()); assertNotEquals(timestamp, r.rawCells()[0].getTimestamp()); r = table.get(new Get(ROW)); assertEquals(1, r.size()); assertNotEquals(timestamp, r.rawCells()[0].getTimestamp()); }
@Test public void testPutMasked() throws IOException { try (Table t = createTable()) { t.put(new Put(ROW).addColumn(FAMILY, col1, 1000001, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000002, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000003, value)); t.put(new Put(ROW).addColumn(FAMILY, col1, 1000004, value)); t.delete(new Delete(ROW).addColumn(FAMILY, col1, 1000003)); Result r = t.get(new Get(ROW).setMaxVersions(3)); assertEquals(2, r.size()); assertEquals(1000004, r.rawCells()[0].getTimestamp()); assertEquals(1000002, r.rawCells()[1].getTimestamp()); TEST_UTIL.getAdmin().flush(t.getName()); r = t.get(new Get(ROW).setMaxVersions(3)); assertEquals(2, r.size()); assertEquals(1000004, r.rawCells()[0].getTimestamp()); assertEquals(1000002, r.rawCells()[1].getTimestamp()); } }
private void verifyData( final HRegion newReg, final int startRow, final int numRows, final byte[]... families) throws IOException { for (int i = startRow; i < startRow + numRows; i++) { byte[] row = Bytes.toBytes("" + i); Get get = new Get(row); Result result = newReg.get(get); Cell[] raw = result.rawCells(); assertEquals(families.length, result.size()); for (int j = 0; j < families.length; j++) { assertTrue(CellUtil.matchingRows(raw[j], row)); assertTrue(CellUtil.matchingFamily(raw[j], families[j])); } } }