private static long[] scanResultList(long rows, int threshold, int extra) { // 1. Simulate emitting all records in closeRecordProcessor(). long t1 = scanResultList(rows, 0, 0, null, "a", "b"); // 2. Simulate emitting records in processNextRecord() with small memory usage limit. long t2 = scanResultList(rows, threshold, 0, null, "c", "d"); // 3. Simulate emitting records in processNextRecord() with large memory usage limit. long t3 = scanResultList(rows, threshold * 10, 0, null, "e", "f"); // 4. Same as 2. Also emit extra records from a separate thread. long t4 = scanResultList(rows, threshold, extra, null, "g", "h"); // 5. Same as 3. Also emit extra records from a separate thread. long t5 = scanResultList(rows, threshold * 10, extra, null, "i", "j"); return new long[] {t1, t2, t3, t4, t5}; }
scanResultList(rows, threshold, extra); long[] tmp = scanResultList(rows, threshold, extra); for (int k = 0; k < count; k++) { t[k] += tmp[k];
private static void scanAndVerify( long rows, int threshold, int separate, String prefix1, String prefix2) { ArrayList<Tuple2<HiveKey, BytesWritable>> output = new ArrayList<Tuple2<HiveKey, BytesWritable>>((int)rows); scanResultList(rows, threshold, separate, output, prefix1, prefix2); assertEquals(rows, output.size()); long primaryRows = rows * (100 - separate) / 100; long separateRows = rows - primaryRows; HashSet<Long> primaryRowKeys = new HashSet<Long>(); HashSet<Long> separateRowKeys = new HashSet<Long>(); for (Tuple2<HiveKey, BytesWritable> item: output) { String key = bytesWritableToString(item._1); String value = bytesWritableToString(item._2); String prefix = key.substring(0, key.indexOf('_')); Long id = Long.valueOf(key.substring(5 + prefix.length())); if (prefix.equals(prefix1)) { assertTrue(id >= 0 && id < primaryRows); primaryRowKeys.add(id); } else { assertEquals(prefix2, prefix); assertTrue(id >= 0 && id < separateRows); separateRowKeys.add(id); } assertEquals(prefix + "_value_" + id, value); } assertEquals(separateRows, separateRowKeys.size()); assertEquals(primaryRows, primaryRowKeys.size()); }