@VisibleForTesting public void testPutRow(long currentKey) throws HiveException, IOException { add(currentKey, null); }
@Override public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws HiveException, IOException { // Ignore NULL keys (HashSet not used for FULL OUTER). adaptPutRow(currentKey, currentValue); }
public void verify(VectorMapJoinFastLongHashSet map) { int mapSize = map.size(); if (mapSize != count) { TestCase.fail("map.size() does not match expected count"); } for (int index = 0; index < count; index++) { FastLongHashSetElement element = array[index]; long key = element.getKey(); VectorMapJoinHashSetResult hashSetResult = map.createHashSetResult(); JoinUtil.JoinResult joinResult = map.contains(key, hashSetResult); if (joinResult != JoinUtil.JoinResult.MATCH) { assertTrue(false); } } } }
@Test public void testGetNonExistent() throws Exception { random = new Random(100); VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); long key1 = random.nextLong(); map.testPutRow(key1); verifyTable.add(key1); verifyTable.verify(map); long key2 = key1 += 1; VectorMapJoinHashSetResult hashSetResult = map.createHashSetResult(); JoinUtil.JoinResult joinResult = map.contains(key2, hashSetResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); map.testPutRow(key2); verifyTable.add(key2); verifyTable.verify(map); long key3 = key2 += 1; hashSetResult = map.createHashSetResult(); joinResult = map.contains(key3, hashSetResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); }
@Test public void testMultipleKeysMultipleValue() throws Exception { random = new Random(7778); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); int keyCount = 1000; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); }
@Test public void testOneKey() throws Exception { random = new Random(4186); VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); long key = random.nextLong(); map.testPutRow(key); verifyTable.add(key); verifyTable.verify(map); // Second time. map.testPutRow(key); verifyTable.add(key); verifyTable.verify(map); // Third time. map.testPutRow(key); verifyTable.add(key); verifyTable.verify(map); }
@Override public JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) { VectorMapJoinFastHashSet.HashSetResult optimizedHashSetResult = (VectorMapJoinFastHashSet.HashSetResult) hashSetResult; optimizedHashSetResult.forget(); long hashCode = HashCodeUtil.calculateLongHashCode(key); long existance = findReadSlot(key, hashCode); JoinUtil.JoinResult joinResult; if (existance == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashSetResult.setJoinResult(joinResult); return joinResult; }
new VectorMapJoinFastLongHashSet( false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, -1); map.testPutRow(key); verifyTable.add(key); VectorMapJoinHashSetResult hashSetResult = map.createHashSetResult(); JoinUtil.JoinResult joinResult = map.contains(anotherKey, hashSetResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH);
@Test public void testLargeAndExpand() throws Exception { random = new Random(56); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); int keyCount = 1000; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); } }
@Test public void testMultipleKeysSingleValue() throws Exception { random = new Random(1412); VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); int keyCount = 100 + random.nextInt(1000); for (int i = 0; i < keyCount; i++) { long key; while (true) { key = random.nextLong(); if (!verifyTable.contains(key)) { // Unique keys for this test. break; } } map.testPutRow(key); verifyTable.add(key); // verifyTable.verify(map); } verifyTable.verify(map); }
@Override public JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) { VectorMapJoinFastHashSet.HashSetResult optimizedHashSetResult = (VectorMapJoinFastHashSet.HashSetResult) hashSetResult; optimizedHashSetResult.forget(); long hashCode = HashCodeUtil.calculateLongHashCode(key); int pairIndex = findReadSlot(key, hashCode); JoinUtil.JoinResult joinResult; if (pairIndex == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { /* * NOTE: Support for trackMatched not needed yet for Set. if (matchTracker != null) { matchTracker.trackMatch(pairIndex / 2); } */ joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashSetResult.setJoinResult(joinResult); return joinResult; }
break; case HASH_SET: hashTable = new VectorMapJoinFastLongHashSet( isFullOuter, minMaxEnabled,
@Test public void testExpand() throws Exception { random = new Random(348); // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); for (int i = 0; i < 18; ++i) { long key; while (true) { key = random.nextLong(); if (!verifyTable.contains(key)) { // Unique keys for this test. break; } } map.testPutRow(key); verifyTable.add(key); // verifyTable.verify(map); } verifyTable.verify(map); // assertEquals(1 << 18, map.getCapacity()); }
@Override public JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) { VectorMapJoinFastHashSet.HashSetResult optimizedHashSetResult = (VectorMapJoinFastHashSet.HashSetResult) hashSetResult; optimizedHashSetResult.forget(); long hashCode = VectorMapJoinFastLongHashUtil.hashKey(key); long existance = findReadSlot(key, hashCode); JoinUtil.JoinResult joinResult; if (existance == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashSetResult.setJoinResult(joinResult); return joinResult; }
@VisibleForTesting public void testPutRow(long currentKey) throws HiveException, IOException { add(currentKey, null); }
break; case HASH_SET: hashTable = new VectorMapJoinFastLongHashSet( minMaxEnabled, isOuterJoin, hashTableKeyType, newThreshold, loadFactor, writeBufferSize, estimatedKeyCount);
break; case HASH_SET: hashTable = new VectorMapJoinFastLongHashSet( minMaxEnabled, isOuterJoin, hashTableKeyType, newThreshold, loadFactor, writeBufferSize);