public void hintUnmatchedOrdinalCount(String type, int numOrdinals) { unmatchedOrdinalRemapping.put(type, new IntMap(numOrdinals)); }
private void populateStats() { iter.reset(); int removedEntryCount = 0; long totalElementCount = 0; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; totalElementCount += typeState.size(ordinal); ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; historicalDataElements.totalNumberOfElements = totalElementCount; historicalDataElements.bitsPerListPointer = totalElementCount == 0 ? 1 : 64 - Long.numberOfLeadingZeros(totalElementCount); historicalDataElements.bitsPerElement = stateEngineDataElements[0].bitsPerElement; ordinalMapping = new IntMap(removedEntryCount); }
private void populateStats() { iter.reset(); int removedEntryCount = 0; int maxSize = 0; long totalBucketCount = 0; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; int size = typeState.size(ordinal); if(size > maxSize) maxSize = size; totalBucketCount += HashCodes.hashTableSize(size); ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; historicalDataElements.bitsPerSetPointer = 64 - Long.numberOfLeadingZeros(totalBucketCount); historicalDataElements.bitsPerSetSizeValue = 64 - Long.numberOfLeadingZeros(maxSize); historicalDataElements.bitsPerFixedLengthSetPortion = historicalDataElements.bitsPerSetPointer + historicalDataElements.bitsPerSetSizeValue; historicalDataElements.bitsPerElement = stateEngineDataElements[0].bitsPerElement; historicalDataElements.emptyBucketValue = stateEngineDataElements[0].emptyBucketValue; historicalDataElements.totalNumberOfBuckets = totalBucketCount; ordinalMapping = new IntMap(removedEntryCount); }
private void populateStats() { iter.reset(); int removedEntryCount = 0; long totalVarLengthSizes[] = new long[stateEngineDataElements[0].varLengthData.length]; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; for(int i=0;i<totalVarLengthSizes.length;i++) { if(stateEngineDataElements[0].varLengthData[i] != null) { totalVarLengthSizes[i] += varLengthSize(ordinal, i); } } ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; for(int i=0;i<stateEngineDataElements[0].bitsPerField.length;i++) { if(stateEngineDataElements[0].varLengthData[i] == null) { historicalDataElements.bitsPerField[i] = stateEngineDataElements[0].bitsPerField[i]; } else { historicalDataElements.bitsPerField[i] = (64 - Long.numberOfLeadingZeros(totalVarLengthSizes[i] + 1)) + 1; } historicalDataElements.nullValueForField[i] = (1L << historicalDataElements.bitsPerField[i]) - 1; historicalDataElements.bitOffsetPerField[i] = historicalDataElements.bitsPerRecord; historicalDataElements.bitsPerRecord += historicalDataElements.bitsPerField[i]; } ordinalMapping = new IntMap(removedEntryCount); }
private void populateStats() { iter.reset(); int removedEntryCount = 0; int maxSize = 0; long totalBucketCount = 0; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; int size = typeState.size(ordinal); if(size > maxSize) maxSize = size; totalBucketCount += HashCodes.hashTableSize(size); ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; historicalDataElements.bitsPerMapPointer = 64 - Long.numberOfLeadingZeros(totalBucketCount); historicalDataElements.bitsPerMapSizeValue = 64 - Long.numberOfLeadingZeros(maxSize); historicalDataElements.bitsPerFixedLengthMapPortion = historicalDataElements.bitsPerMapPointer + historicalDataElements.bitsPerMapSizeValue; historicalDataElements.bitsPerKeyElement = stateEngineDataElements[0].bitsPerKeyElement; historicalDataElements.bitsPerValueElement = stateEngineDataElements[0].bitsPerValueElement; historicalDataElements.bitsPerMapEntry = stateEngineDataElements[0].bitsPerMapEntry; historicalDataElements.emptyBucketKeyValue = stateEngineDataElements[0].emptyBucketKeyValue; historicalDataElements.totalNumberOfBuckets = totalBucketCount; ordinalMapping = new IntMap(removedEntryCount); }
private IntMap copyAllRecords(HollowTypeReadState typeState, DiffEqualityMappingOrdinalRemapper ordinalRemapper, HollowWriteStateEngine writeEngine) { String typeName = typeState.getSchema().getName(); HollowRecordCopier recordCopier = HollowRecordCopier.createCopier(typeState, ordinalRemapper, false); ///NOTE: This will invalidate custom hash codes PopulatedOrdinalListener listener = typeState.getListener(PopulatedOrdinalListener.class); IntMap ordinalLookupMap = new IntMap(listener.getPopulatedOrdinals().cardinality()); int ordinal = listener.getPopulatedOrdinals().nextSetBit(0); while(ordinal != -1) { HollowWriteRecord rec = recordCopier.copy(ordinal); int mappedOrdinal = writeEngine.add(typeName, rec); ordinalLookupMap.put(ordinal, mappedOrdinal); ordinal = listener.getPopulatedOrdinals().nextSetBit(ordinal + 1); } return ordinalLookupMap; }
IntMap ordinalLookupMap = new IntMap(shouldCopyAllRecords ? previouslyPopulatedOrdinals.cardinality() : unmatchedRecordCount);
private IntMap remapPreviousOrdinalMapping(IntMap previousOrdinalMapping, String typeName, OrdinalRemapper ordinalRemapper) { IntMapEntryIterator ordinalMappingIter = previousOrdinalMapping.iterator(); IntMap ordinalLookupMap = new IntMap(previousOrdinalMapping.size()); while(ordinalMappingIter.next()) ordinalLookupMap.put(ordinalRemapper.getMappedOrdinal(typeName, ordinalMappingIter.getKey()), ordinalMappingIter.getValue()); return ordinalLookupMap; }
public HollowHistoricalStateTypeKeyOrdinalMapping remap(OrdinalRemapper remapper) { IntMap newAddedOrdinalMap = new IntMap(addedOrdinalMap.size()); IntMapEntryIterator addedIter = addedOrdinalMap.iterator(); while(addedIter.next()) newAddedOrdinalMap.put(addedIter.getKey(), remapper.getMappedOrdinal(typeName, addedIter.getValue())); IntMap newRemovedOrdinalMap = new IntMap(removedOrdinalMap.size()); IntMapEntryIterator removedIter = removedOrdinalMap.iterator(); while(removedIter.next()) newRemovedOrdinalMap.put(removedIter.getKey(), remapper.getMappedOrdinal(typeName, removedIter.getValue())); return new HollowHistoricalStateTypeKeyOrdinalMapping(typeName, keyIndex, newAddedOrdinalMap, newRemovedOrdinalMap); }
private void remapTheChangedDataToUnusedOrdinals() { PartialOrdinalRemapper remapper = new PartialOrdinalRemapper(); for(HollowSchema schema : schemas) { BitSet ordinalsToRemap = changedOrdinalsBetweenStates.get(schema.getName()); HollowTypeReadState fromTypeState = from.getTypeState(schema.getName()); HollowTypeReadState toTypeState = to.getTypeState(schema.getName()); HollowTypeWriteState typeWriteState = writeEngine.getTypeState(schema.getName()); IntMap ordinalRemapping = new IntMap(ordinalsToRemap.cardinality()); int nextFreeOrdinal = Math.max(fromTypeState.maxOrdinal(), toTypeState.maxOrdinal()) + 1; boolean preserveHashPositions = shouldPreserveHashPositions(schema); HollowRecordCopier copier = HollowRecordCopier.createCopier(fromTypeState, schema, remapper, preserveHashPositions); int ordinal = ordinalsToRemap.nextSetBit(0); while(ordinal != -1 && ordinal <= fromTypeState.maxOrdinal()) { HollowWriteRecord copy = copier.copy(ordinal); typeWriteState.mapOrdinal(copy, nextFreeOrdinal, false, true); ordinalRemapping.put(ordinal, nextFreeOrdinal++); ordinal = ordinalsToRemap.nextSetBit(ordinal + 1); } remapper.addOrdinalRemapping(schema.getName(), ordinalRemapping); typeWriteState.recalculateFreeOrdinals(); } }