public HollowSetWriteRecord(HashBehavior defaultHashBehavior) { this.elementsAndHashes = new LongList(); this.defaultHashBehavior = defaultHashBehavior; }
public void addElement(int ordinal, int hashCode) { long elementAndHash = (long)ordinal << 32 | (hashCode & 0xFFFFFFFFL); elementsAndHashes.add(elementAndHash); }
@Override public void writeDataTo(ByteDataBuffer buf, HashBehavior hashBehavior) { elementsAndHashes.sort(); int hashTableSize = HashCodes.hashTableSize(elementsAndHashes.size()); int bucketMask = hashTableSize - 1; /// hashTableSize is a power of 2. VarInt.writeVInt(buf, elementsAndHashes.size()); int previousOrdinal = 0; for(int i=0;i<elementsAndHashes.size();i++) { int ordinal = (int)(elementsAndHashes.get(i) >>> 32); VarInt.writeVInt(buf, ordinal - previousOrdinal); if(hashBehavior != IGNORED_HASHES) { int hashCode = (int)elementsAndHashes.get(i); if(hashBehavior == MIXED_HASHES) hashCode = HashCodes.hashInt(hashCode); int bucketToHashTo = hashCode & bucketMask; VarInt.writeVInt(buf, bucketToHashTo); } previousOrdinal = ordinal; } }
private void indexOrdinal(HollowObjectTypeReadState keyTypeState, int ordinal, int[] hashedRecordKeys, int[][] hashedFieldKeys, LongList hashedFieldKeyChains) { int bucketMask = hashedRecordKeys.length - 1; int bucket = hashKeyRecord(keyTypeState, ordinal) & bucketMask; while(hashedRecordKeys[bucket] != -1) bucket = (bucket + 1) & bucketMask; hashedRecordKeys[bucket] = ordinal; indexFields: for(int j=0;j<primaryKey.numFields();j++) { if(keyFieldIsIndexed[j]) { int fieldBucket = HashCodes.hashInt(HollowReadFieldUtils.fieldHashCode(keyTypeState, ordinal, j)) & bucketMask; int chainStartIndex = hashedFieldKeys[j][fieldBucket]; while(chainStartIndex != -1) { int representativeOrdinal = (int)hashedFieldKeyChains.get(chainStartIndex); if(HollowReadFieldUtils.fieldsAreEqual(keyTypeState, ordinal, j, keyTypeState, representativeOrdinal, j)) { hashedFieldKeyChains.add(((long)chainStartIndex << 32) | ordinal); hashedFieldKeys[j][fieldBucket] = hashedFieldKeyChains.size() - 1; break indexFields; } fieldBucket = (fieldBucket + 1) & bucketMask; chainStartIndex = hashedFieldKeys[j][fieldBucket]; } hashedFieldKeyChains.add(((long)Integer.MAX_VALUE << 32) | ordinal); hashedFieldKeys[j][fieldBucket] = hashedFieldKeyChains.size() - 1; } } }
public boolean recordsAreEqual(int ordinal) { int size = fromMapState.size(ordinal); if(toMapState.size(ordinal) != size) return false; fromScratch.clear(); toScratch.clear(); HollowMapEntryOrdinalIterator iter = fromMapState.ordinalIterator(ordinal); while(iter.next()) fromScratch.add((iter.getKey() << 32) | iter.getValue()); iter = toMapState.ordinalIterator(ordinal); while(iter.next()) toScratch.add((iter.getKey() << 32) | iter.getValue()); fromScratch.sort(); toScratch.sort(); return fromScratch.equals(toScratch); } };
matchPairResults[threadNumber] = new LongList(); executor.execute(() -> { EqualityDeterminer equalityDeterminer = getEqualityDeterminer(); while(hashedToOrdinals[bucket] != -1) { if(equalityDeterminer.recordsAreEqual(t, hashedToOrdinals[bucket])) { matchPairResults[threadNumber].add(((long) t << 32) | hashedToOrdinals[bucket]); numMatches += matchPairResults[i].size();
/** * @return the total number of matched records (based on primary key) */ public int getTotalNumberOfMatches() { return matcher.getMatchedOrdinals().size(); }
private void addMatches(Matcher matcher, int fieldIndex, int hashCode, IntList results) { int hashIntCode = HashCodes.hashInt(hashCode); int bucket = hashIntCode & (hashedFieldKeys[fieldIndex].length - 1); while(hashedFieldKeys[fieldIndex][bucket] != -1) { int chainIndex = hashedFieldKeys[fieldIndex][bucket]; int representativeOrdinal = (int)hashedFieldKeyChains.get(chainIndex); if(matcher.foundMatch(representativeOrdinal)) { while(representativeOrdinal != -1) { results.add(representativeOrdinal); chainIndex = (int)(hashedFieldKeyChains.get(chainIndex) >> 32); representativeOrdinal = (chainIndex == Integer.MAX_VALUE) ? -1 : (int)hashedFieldKeyChains.get(chainIndex); } return; } bucket++; bucket &= hashedFieldKeys[fieldIndex].length - 1; } }
@Override public void reset() { elementsAndHashes.clear(); }
public boolean next() { list.clear(); while(currentShardList < shardedResults.length) { if(currentShardListPosition < shardedResults[currentShardList].size()) { currentFromOrdinal = (int)(shardedResults[currentShardList].get(currentShardListPosition) >> 32); while(currentShardListPosition < shardedResults[currentShardList].size() && (int)(shardedResults[currentShardList].get(currentShardListPosition) >> 32) == currentFromOrdinal) { int toOrdinal = (int)shardedResults[currentShardList].get(currentShardListPosition); list.add(toOrdinal); currentShardListPosition++; } return true; } currentShardListPosition = 0; currentShardList++; } return false; }
public void calculateMatches() { fromIdx = new HollowPrimaryKeyIndex(fromTypeState.getStateEngine(), fromTypeState.getSchema().getName(), matchPaths.toArray(new String[matchPaths.size()])); toIdx = new HollowPrimaryKeyIndex(toTypeState.getStateEngine(), toTypeState.getSchema().getName(), matchPaths.toArray(new String[matchPaths.size()])); BitSet fromPopulatedOrdinals = fromTypeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals(); BitSet fromUnmatchedOrdinals = new BitSet(fromPopulatedOrdinals.length()); fromUnmatchedOrdinals.or(fromPopulatedOrdinals); BitSet toPopulatedOrdinals = toTypeState.getListener(PopulatedOrdinalListener.class).getPopulatedOrdinals(); int candidateToMatchOrdinal = toPopulatedOrdinals.nextSetBit(0); while(candidateToMatchOrdinal != -1) { Object key[] = toIdx.getRecordKey(candidateToMatchOrdinal); int matchedOrdinal = fromIdx.getMatchingOrdinal(key); if(matchedOrdinal != -1) { matchedOrdinals.add(((long)matchedOrdinal << 32) | candidateToMatchOrdinal); fromUnmatchedOrdinals.clear(matchedOrdinal); } else { extraInTo.add(candidateToMatchOrdinal); } candidateToMatchOrdinal = toPopulatedOrdinals.nextSetBit(candidateToMatchOrdinal + 1); } int unmatchedFromOrdinal = fromUnmatchedOrdinals.nextSetBit(0); while(unmatchedFromOrdinal != -1) { extraInFrom.add(unmatchedFromOrdinal); unmatchedFromOrdinal = fromUnmatchedOrdinals.nextSetBit(unmatchedFromOrdinal + 1); } }
@Override public void run() { DiffEqualityMapping equalityMapping = rootDiff.getEqualityMapping(); HollowDiffCountingNode rootNode = new HollowDiffObjectCountingNode(rootDiff, HollowTypeDiff.this, rootId, from, to); DiffEqualOrdinalMap rootNodeOrdinalMap = equalityMapping.getEqualOrdinalMap(type); boolean requiresMissingFieldTraversal = equalityMapping.requiresMissingFieldTraversal(type); LongList matches = matcher.getMatchedOrdinals(); for(int i=threadId;i<matches.size();i+=numThreads) { int fromOrdinal = (int)(matches.get(i) >> 32); int toOrdinal = (int)matches.get(i); if(rootNodeOrdinalMap.getIdentityFromOrdinal(fromOrdinal) == -1 || rootNodeOrdinalMap.getIdentityFromOrdinal(fromOrdinal) != rootNodeOrdinalMap.getIdentityToOrdinal(toOrdinal)) { rootNode.prepare(fromOrdinal, toOrdinal); rootNode.traverseDiffs(fromIntList(fromOrdinal), toIntList(toOrdinal)); } else if(requiresMissingFieldTraversal) { rootNode.prepare(fromOrdinal, toOrdinal); rootNode.traverseMissingFields(fromIntList(fromOrdinal), toIntList(toOrdinal)); } } results[threadId] = rootNode.getFieldDiffs(); }
public HollowDiffMatcher(HollowObjectTypeReadState fromTypeState, HollowObjectTypeReadState toTypeState) { this.matchPaths = new ArrayList<String>(); this.fromTypeState = fromTypeState; this.toTypeState = toTypeState; this.matchedOrdinals = new LongList(); this.extraInFrom = new IntList(); this.extraInTo = new IntList(); }
public void run() { HollowIndexerValueTraverser fromTraverser = new HollowIndexerValueTraverser(from, type, elementPaths); HollowIndexerValueTraverser toTraverser = new HollowIndexerValueTraverser(to, type, elementPaths); int hashedResults[] = new int[16]; for(int i=threadNumber;i<matcher.getMatchedOrdinals().size();i += numThreads) { long ordinalPair = matcher.getMatchedOrdinals().get(i); int fromOrdinal = (int)(ordinalPair >>> 32); int toOrdinal = (int)ordinalPair; fromTraverser.traverse(fromOrdinal); toTraverser.traverse(toOrdinal); if(fromTraverser.getNumMatches() * 2 > hashedResults.length) hashedResults = new int[hashTableSize(fromTraverser.getNumMatches())]; populateHashTable(fromTraverser, hashedResults); countMatches(fromTraverser, toTraverser, hashedResults); } for(int i=threadNumber;i<matcher.getExtraInFrom().size();i+=numThreads) { fromTraverser.traverse(matcher.getExtraInFrom().get(i)); totalUnmatchedFromElements.addAndGet(fromTraverser.getNumMatches()); } for(int i=threadNumber;i<matcher.getExtraInTo().size();i+=numThreads) { toTraverser.traverse(matcher.getExtraInTo().get(i)); totalUnmatchedToElements.addAndGet(toTraverser.getNumMatches()); } } });
private void rehashAllRecordKeys(HollowObjectTypeReadState keyTypeState, int hashTableSize) { int[] hashedRecordKeys = initializeHashedKeyArray(hashTableSize); int[][] hashedFieldKeys = new int[primaryKey.numFields()][]; LongList hashedFieldKeyChains = new LongList(); for(int i=0;i<primaryKey.numFields();i++) if(keyFieldIsIndexed[i]) hashedFieldKeys[i] = initializeHashedKeyArray(hashTableSize); for(int i=0;i<=keyTypeState.maxOrdinal();i++) indexOrdinal(keyTypeState, i, hashedRecordKeys, hashedFieldKeys, hashedFieldKeyChains); this.hashedRecordKeys = hashedRecordKeys; this.hashedFieldKeys = hashedFieldKeys; this.hashedFieldKeyChains = hashedFieldKeyChains; this.maxIndexedKeyOrdinal = keyTypeState.maxOrdinal(); }