String k2 = getRandomString(rand.nextInt(256)); long v1 = rand.nextLong(); long v2 = rand.nextLong(); appendRow(batch, makeKeyRow(k1, k2), makeValueRow(v1, v2)); expectedK1[i] = k1; expectedK2[i] = k2; if (rand.nextBoolean()) { UnsafeRow key = batch.getKeyRow(rowId); Assert.assertTrue(checkKey(key, expectedK1[rowId], expectedK2[rowId])); Assert.assertTrue(checkValue(value, expectedV1[rowId], expectedV2[rowId]));
@Test public void setUpdateAndRetrieve() { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertEquals(1, batch.numRows()); UnsafeRow retrievedValue = batch.getValueRow(0); updateValueRow(retrievedValue, 2, 2); UnsafeRow retrievedValue2 = batch.getValueRow(0); Assert.assertTrue(checkValue(retrievedValue2, 2, 2)); } finally { batch.close(); } }
@Test public void failureToAllocateFirstPage() throws Exception { memoryManager.limit(1024); RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(11, 11); UnsafeRow ret = appendRow(batch, key, value); Assert.assertNull(ret); Assert.assertFalse(batch.rowIterator().next()); } finally { batch.close(); } }
@Test public void failureToAllocateFirstPage() throws Exception { memoryManager.limit(1024); RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(11, 11); UnsafeRow ret = appendRow(batch, key, value); Assert.assertNull(ret); Assert.assertFalse(batch.rowIterator().next()); } finally { batch.close(); } }
String k2 = getRandomString(rand.nextInt(256)); long v1 = rand.nextLong(); long v2 = rand.nextLong(); appendRow(batch, makeKeyRow(k1, k2), makeValueRow(v1, v2)); expectedK1[i] = k1; expectedK2[i] = k2; if (rand.nextBoolean()) { UnsafeRow key = batch.getKeyRow(rowId); Assert.assertTrue(checkKey(key, expectedK1[rowId], expectedK2[rowId])); Assert.assertTrue(checkValue(value, expectedV1[rowId], expectedV2[rowId]));
@Test public void setUpdateAndRetrieve() { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertEquals(1, batch.numRows()); UnsafeRow retrievedValue = batch.getValueRow(0); updateValueRow(retrievedValue, 2, 2); UnsafeRow retrievedValue2 = batch.getValueRow(0); Assert.assertTrue(checkValue(retrievedValue2, 2, 2)); } finally { batch.close(); } }
@Test public void failureToAllocateFirstPage() throws Exception { memoryManager.limit(1024); RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(11, 11); UnsafeRow ret = appendRow(batch, key, value); Assert.assertNull(ret); Assert.assertFalse(batch.rowIterator().next()); } finally { batch.close(); } }
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow ret1 = appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertTrue(checkValue(ret1, 1, 1)); UnsafeRow ret2 = appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); Assert.assertTrue(checkValue(ret2, 2, 2)); UnsafeRow ret3 = appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertTrue(checkValue(ret3, 3, 3)); Assert.assertEquals(3, batch.numRows()); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 1, "A")); UnsafeRow retrievedKey2 = batch.getKeyRow(1); Assert.assertTrue(checkKey(retrievedKey2, 2, "B")); UnsafeRow retrievedValue1 = batch.getValueRow(1); Assert.assertTrue(checkValue(retrievedValue1, 2, 2)); UnsafeRow retrievedValue2 = batch.getValueRow(2); Assert.assertTrue(checkValue(retrievedValue2, 3, 3)); try { batch.getKeyRow(3);
@Test public void setUpdateAndRetrieve() { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertEquals(1, batch.numRows()); UnsafeRow retrievedValue = batch.getValueRow(0); updateValueRow(retrievedValue, 2, 2); UnsafeRow retrievedValue2 = batch.getValueRow(0); Assert.assertTrue(checkValue(retrievedValue2, 2, 2)); } finally { batch.close(); } }
String k2 = getRandomString(rand.nextInt(256)); long v1 = rand.nextLong(); long v2 = rand.nextLong(); appendRow(batch, makeKeyRow(k1, k2), makeValueRow(v1, v2)); expectedK1[i] = k1; expectedK2[i] = k2; if (rand.nextBoolean()) { UnsafeRow key = batch.getKeyRow(rowId); Assert.assertTrue(checkKey(key, expectedK1[rowId], expectedK2[rowId])); Assert.assertTrue(checkValue(value, expectedV1[rowId], expectedV2[rowId]));
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow ret1 = appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertTrue(checkValue(ret1, 1, 1)); UnsafeRow ret2 = appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); Assert.assertTrue(checkValue(ret2, 2, 2)); UnsafeRow ret3 = appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertTrue(checkValue(ret3, 3, 3)); Assert.assertEquals(3, batch.numRows()); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 1, "A")); UnsafeRow retrievedKey2 = batch.getKeyRow(1); Assert.assertTrue(checkKey(retrievedKey2, 2, "B")); UnsafeRow retrievedValue1 = batch.getValueRow(1); Assert.assertTrue(checkValue(retrievedValue1, 2, 2)); UnsafeRow retrievedValue2 = batch.getValueRow(2); Assert.assertTrue(checkValue(retrievedValue2, 3, 3)); try { batch.getKeyRow(3);
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow ret1 = appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertTrue(checkValue(ret1, 1, 1)); UnsafeRow ret2 = appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); Assert.assertTrue(checkValue(ret2, 2, 2)); UnsafeRow ret3 = appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertTrue(checkValue(ret3, 3, 3)); Assert.assertEquals(3, batch.numRows()); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 1, "A")); UnsafeRow retrievedKey2 = batch.getKeyRow(1); Assert.assertTrue(checkKey(retrievedKey2, 2, "B")); UnsafeRow retrievedValue1 = batch.getValueRow(1); Assert.assertTrue(checkValue(retrievedValue1, 2, 2)); UnsafeRow retrievedValue2 = batch.getValueRow(2); Assert.assertTrue(checkValue(retrievedValue2, 3, 3)); try { batch.getKeyRow(3);
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); Assert.assertTrue(iterator.next()); UnsafeRow key2 = iterator.getKey(); UnsafeRow value2 = iterator.getValue(); Assert.assertTrue(checkKey(key2, 2, "B")); Assert.assertTrue(checkValue(value2, 2, 2)); Assert.assertTrue(iterator.next()); UnsafeRow key3 = iterator.getKey(); UnsafeRow value3 = iterator.getValue(); Assert.assertTrue(checkKey(key3, 3, "C")); Assert.assertTrue(checkValue(value3, 3, 3)); Assert.assertFalse(iterator.next()); } finally {
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); Assert.assertTrue(iterator.next()); UnsafeRow key2 = iterator.getKey(); UnsafeRow value2 = iterator.getValue(); Assert.assertTrue(checkKey(key2, 2, "B")); Assert.assertTrue(checkValue(value2, 2, 2)); Assert.assertTrue(iterator.next()); UnsafeRow key3 = iterator.getKey(); UnsafeRow value3 = iterator.getValue(); Assert.assertTrue(checkKey(key3, 3, "C")); Assert.assertTrue(checkValue(value3, 3, 3)); Assert.assertFalse(iterator.next()); } finally {
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); Assert.assertTrue(iterator.next()); UnsafeRow key2 = iterator.getKey(); UnsafeRow value2 = iterator.getValue(); Assert.assertTrue(checkKey(key2, 2, "B")); Assert.assertTrue(checkValue(value2, 2, 2)); Assert.assertTrue(iterator.next()); UnsafeRow key3 = iterator.getKey(); UnsafeRow value3 = iterator.getValue(); Assert.assertTrue(checkKey(key3, 3, "C")); Assert.assertTrue(checkValue(value3, 3, 3)); Assert.assertFalse(iterator.next()); } finally {
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(11, 11), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(22, 22), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(33, 33), makeValueRow(3, 3)); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 11, 11)); UnsafeRow retrievedKey2 = batch.getKeyRow(1); Assert.assertTrue(checkKey(retrievedKey2, 22, 22)); UnsafeRow retrievedValue1 = batch.getValueRow(1); Assert.assertTrue(checkValue(retrievedValue1, 2, 2)); UnsafeRow retrievedValue2 = batch.getValueRow(2); Assert.assertTrue(checkValue(retrievedValue2, 3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 11, 11)); Assert.assertTrue(checkValue(value1, 1, 1)); Assert.assertTrue(iterator.next()); UnsafeRow key2 = iterator.getKey(); UnsafeRow value2 = iterator.getValue(); Assert.assertTrue(checkKey(key2, 22, 22)); Assert.assertTrue(checkValue(value2, 2, 2)); Assert.assertTrue(iterator.next()); UnsafeRow key3 = iterator.getKey(); UnsafeRow value3 = iterator.getValue(); Assert.assertTrue(checkKey(key3, 33, 33));
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(11, 11), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(22, 22), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(33, 33), makeValueRow(3, 3)); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 11, 11)); UnsafeRow retrievedKey2 = batch.getKeyRow(1); Assert.assertTrue(checkKey(retrievedKey2, 22, 22)); UnsafeRow retrievedValue1 = batch.getValueRow(1); Assert.assertTrue(checkValue(retrievedValue1, 2, 2)); UnsafeRow retrievedValue2 = batch.getValueRow(2); Assert.assertTrue(checkValue(retrievedValue2, 3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 11, 11)); Assert.assertTrue(checkValue(value1, 1, 1)); Assert.assertTrue(iterator.next()); UnsafeRow key2 = iterator.getKey(); UnsafeRow value2 = iterator.getValue(); Assert.assertTrue(checkKey(key2, 22, 22)); Assert.assertTrue(checkValue(value2, 2, 2)); Assert.assertTrue(iterator.next()); UnsafeRow key3 = iterator.getKey(); UnsafeRow value3 = iterator.getValue(); Assert.assertTrue(checkKey(key3, 33, 33));
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(11, 11), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(22, 22), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(33, 33), makeValueRow(3, 3)); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 11, 11)); UnsafeRow retrievedKey2 = batch.getKeyRow(1); Assert.assertTrue(checkKey(retrievedKey2, 22, 22)); UnsafeRow retrievedValue1 = batch.getValueRow(1); Assert.assertTrue(checkValue(retrievedValue1, 2, 2)); UnsafeRow retrievedValue2 = batch.getValueRow(2); Assert.assertTrue(checkValue(retrievedValue2, 3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 11, 11)); Assert.assertTrue(checkValue(value1, 1, 1)); Assert.assertTrue(iterator.next()); UnsafeRow key2 = iterator.getKey(); UnsafeRow value2 = iterator.getValue(); Assert.assertTrue(checkKey(key2, 22, 22)); Assert.assertTrue(checkValue(value2, 2, 2)); Assert.assertTrue(iterator.next()); UnsafeRow key3 = iterator.getKey(); UnsafeRow value3 = iterator.getValue(); Assert.assertTrue(checkKey(key3, 33, 33));
@Test public void appendRowUntilExceedingCapacity() throws Exception { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, 10); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(1, 1); for (int i = 0; i < 10; i++) { appendRow(batch, key, value); } UnsafeRow ret = appendRow(batch, key, value); Assert.assertEquals(batch.numRows(), 10); Assert.assertNull(ret); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator = batch.rowIterator(); for (int i = 0; i < 10; i++) { Assert.assertTrue(iterator.next()); UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); } Assert.assertFalse(iterator.next()); } finally { batch.close(); } }
@Test public void appendRowUntilExceedingCapacity() throws Exception { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, 10); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(1, 1); for (int i = 0; i < 10; i++) { appendRow(batch, key, value); } UnsafeRow ret = appendRow(batch, key, value); Assert.assertEquals(batch.numRows(), 10); Assert.assertNull(ret); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator = batch.rowIterator(); for (int i = 0; i < 10; i++) { Assert.assertTrue(iterator.next()); UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); } Assert.assertFalse(iterator.next()); } finally { batch.close(); } }