@Test public void testResultList() throws Exception { scanAndVerify(10000, 0, 0, "a", "b"); scanAndVerify(10000, 511, 0, "a", "b"); scanAndVerify(10000, 511 * 2, 0, "a", "b"); scanAndVerify(10000, 511, 10, "a", "b"); scanAndVerify(10000, 511 * 2, 10, "a", "b"); scanAndVerify(10000, 512, 0, "a", "b"); scanAndVerify(10000, 512 * 2, 0, "a", "b"); scanAndVerify(10000, 512, 3, "a", "b"); scanAndVerify(10000, 512 * 6, 10, "a", "b"); scanAndVerify(10000, 512 * 7, 5, "a", "b"); scanAndVerify(10000, 512 * 9, 19, "a", "b"); scanAndVerify(10000, 1, 0, "a", "b"); scanAndVerify(10000, 1, 1, "a", "b"); }
private static long[] scanResultList(long rows, int threshold, int extra) { // 1. Simulate emitting all records in closeRecordProcessor(). long t1 = scanResultList(rows, 0, 0, null, "a", "b"); // 2. Simulate emitting records in processNextRecord() with small memory usage limit. long t2 = scanResultList(rows, threshold, 0, null, "c", "d"); // 3. Simulate emitting records in processNextRecord() with large memory usage limit. long t3 = scanResultList(rows, threshold * 10, 0, null, "e", "f"); // 4. Same as 2. Also emit extra records from a separate thread. long t4 = scanResultList(rows, threshold, extra, null, "g", "h"); // 5. Same as 3. Also emit extra records from a separate thread. long t5 = scanResultList(rows, threshold * 10, extra, null, "i", "j"); return new long[] {t1, t2, t3, t4, t5}; }
private static void scanAndVerify( long rows, int threshold, int separate, String prefix1, String prefix2) { ArrayList<Tuple2<HiveKey, BytesWritable>> output = new ArrayList<Tuple2<HiveKey, BytesWritable>>((int)rows); scanResultList(rows, threshold, separate, output, prefix1, prefix2); assertEquals(rows, output.size()); long primaryRows = rows * (100 - separate) / 100; long separateRows = rows - primaryRows; HashSet<Long> primaryRowKeys = new HashSet<Long>(); HashSet<Long> separateRowKeys = new HashSet<Long>(); for (Tuple2<HiveKey, BytesWritable> item: output) { String key = bytesWritableToString(item._1); String value = bytesWritableToString(item._2); String prefix = key.substring(0, key.indexOf('_')); Long id = Long.valueOf(key.substring(5 + prefix.length())); if (prefix.equals(prefix1)) { assertTrue(id >= 0 && id < primaryRows); primaryRowKeys.add(id); } else { assertEquals(prefix2, prefix); assertTrue(id >= 0 && id < separateRows); separateRowKeys.add(id); } assertEquals(prefix + "_value_" + id, value); } assertEquals(separateRows, separateRowKeys.size()); assertEquals(primaryRows, primaryRowKeys.size()); }
@Test public void testSpilling() throws Exception { HiveKVResultCache cache = new HiveKVResultCache(); final int recordCount = HiveKVResultCache.IN_MEMORY_NUM_ROWS * 3; // Test using the same cache where first n rows are inserted then cache is cleared. // Next reuse the same cache and insert another m rows and verify the cache stores correctly. // This simulates reusing the same cache over and over again. testSpillingHelper(cache, recordCount); testSpillingHelper(cache, 1); testSpillingHelper(cache, recordCount); }
scanResultList(rows, threshold, extra); long[] tmp = scanResultList(rows, threshold, extra); for (int k = 0; k < count; k++) { t[k] += tmp[k];