private void populateNewCurrentRecordKeys(HollowObjectTypeReadState typeState) { HollowObjectWriteRecord rec = new HollowObjectWriteRecord(keySchema); PopulatedOrdinalListener listener = typeState.getListener(PopulatedOrdinalListener.class); BitSet populatedOrdinals = listener.getPopulatedOrdinals(); BitSet previousOrdinals = listener.getPreviousOrdinals(); RemovedOrdinalIterator iter = new RemovedOrdinalIterator(populatedOrdinals, previousOrdinals); int ordinal = iter.next(); while(ordinal != -1) { writeKeyObject(typeState, ordinal, rec); ordinal = iter.next(); } }
private HollowHistoricalStateKeyOrdinalMapping createKeyOrdinalMappingFromDelta() { HollowHistoricalStateKeyOrdinalMapping keyOrdinalMapping = new HollowHistoricalStateKeyOrdinalMapping(keyIndex); for(String keyType : keyIndex.getTypeKeyIndexes().keySet()) { HollowHistoricalStateTypeKeyOrdinalMapping typeMapping = keyOrdinalMapping.getTypeMapping(keyType); HollowObjectTypeReadState typeState = (HollowObjectTypeReadState) latestHollowReadStateEngine.getTypeState(keyType); if (typeState==null) { // The type is present in the history's primary key index but is not present // in the latest read state; ensure the mapping is initialized to the default state typeMapping.prepare(0, 0); typeMapping.finish(); continue; } PopulatedOrdinalListener listener = typeState.getListener(PopulatedOrdinalListener.class); RemovedOrdinalIterator removalIterator = new RemovedOrdinalIterator(listener); RemovedOrdinalIterator additionsIterator = new RemovedOrdinalIterator(listener.getPopulatedOrdinals(), listener.getPreviousOrdinals()); typeMapping.prepare(additionsIterator.countTotal(), removalIterator.countTotal()); int removedOrdinal = removalIterator.next(); while(removedOrdinal != -1) { typeMapping.removed(typeState, removedOrdinal); removedOrdinal = removalIterator.next(); } int addedOrdinal = additionsIterator.next(); while(addedOrdinal != -1) { typeMapping.added(typeState, addedOrdinal); addedOrdinal = additionsIterator.next(); } typeMapping.finish(); } return keyOrdinalMapping; }
public HollowMapDeltaHistoricalStateCreator(HollowMapTypeReadState typeState) { this.typeState = typeState; this.stateEngineDataElements = typeState.currentDataElements(); this.historicalDataElements = new HollowMapTypeDataElements(WastefulRecycler.DEFAULT_INSTANCE); this.iter = new RemovedOrdinalIterator(typeState.getListener(PopulatedOrdinalListener.class)); this.shardNumberMask = stateEngineDataElements.length - 1; this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(stateEngineDataElements.length); }
public HollowListDeltaHistoricalStateCreator(HollowListTypeReadState typeState) { this.typeState = typeState; this.stateEngineDataElements = typeState.currentDataElements(); this.historicalDataElements = new HollowListTypeDataElements(WastefulRecycler.DEFAULT_INSTANCE); this.iter = new RemovedOrdinalIterator(typeState.getListener(PopulatedOrdinalListener.class)); this.shardNumberMask = stateEngineDataElements.length - 1; this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(stateEngineDataElements.length); }
private void populateStats() { iter.reset(); int removedEntryCount = 0; long totalElementCount = 0; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; totalElementCount += typeState.size(ordinal); ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; historicalDataElements.totalNumberOfElements = totalElementCount; historicalDataElements.bitsPerListPointer = totalElementCount == 0 ? 1 : 64 - Long.numberOfLeadingZeros(totalElementCount); historicalDataElements.bitsPerElement = stateEngineDataElements[0].bitsPerElement; ordinalMapping = new IntMap(removedEntryCount); }
@Test public void iteratesOverRemovedOrdinals() { BitSet previousOrdinals = bitSet(1, 2, 3, 4, 6, 7, 9, 10); BitSet currentOrdinals = bitSet(1, 3, 4, 5, 7, 8, 9); RemovedOrdinalIterator iter = new RemovedOrdinalIterator(previousOrdinals, currentOrdinals); Assert.assertEquals(2, iter.next()); Assert.assertEquals(6, iter.next()); Assert.assertEquals(10, iter.next()); Assert.assertEquals(-1, iter.next()); }
public HollowSetDeltaHistoricalStateCreator(HollowSetTypeReadState typeState) { this.typeState = typeState; this.stateEngineDataElements = typeState.currentDataElements(); this.historicalDataElements = new HollowSetTypeDataElements(WastefulRecycler.DEFAULT_INSTANCE); this.iter = new RemovedOrdinalIterator(typeState.getListener(PopulatedOrdinalListener.class)); this.shardNumberMask = stateEngineDataElements.length - 1; this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(stateEngineDataElements.length); }
private void populateStats() { iter.reset(); int removedEntryCount = 0; long totalVarLengthSizes[] = new long[stateEngineDataElements[0].varLengthData.length]; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; for(int i=0;i<totalVarLengthSizes.length;i++) { if(stateEngineDataElements[0].varLengthData[i] != null) { totalVarLengthSizes[i] += varLengthSize(ordinal, i); } } ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; for(int i=0;i<stateEngineDataElements[0].bitsPerField.length;i++) { if(stateEngineDataElements[0].varLengthData[i] == null) { historicalDataElements.bitsPerField[i] = stateEngineDataElements[0].bitsPerField[i]; } else { historicalDataElements.bitsPerField[i] = (64 - Long.numberOfLeadingZeros(totalVarLengthSizes[i] + 1)) + 1; } historicalDataElements.nullValueForField[i] = (1L << historicalDataElements.bitsPerField[i]) - 1; historicalDataElements.bitOffsetPerField[i] = historicalDataElements.bitsPerRecord; historicalDataElements.bitsPerRecord += historicalDataElements.bitsPerField[i]; } ordinalMapping = new IntMap(removedEntryCount); }
public HollowObjectDeltaHistoricalStateCreator(HollowObjectTypeReadState typeState) { this.typeState = typeState; this.stateEngineDataElements = typeState.currentDataElements(); this.historicalDataElements = new HollowObjectTypeDataElements(typeState.getSchema(), WastefulRecycler.DEFAULT_INSTANCE); this.iter = new RemovedOrdinalIterator(typeState.getListener(PopulatedOrdinalListener.class)); this.currentWriteVarLengthDataPointers = new long[typeState.getSchema().numFields()]; this.shardNumberMask = stateEngineDataElements.length - 1; this.shardOrdinalShift = 31 - Integer.numberOfLeadingZeros(stateEngineDataElements.length); }
public void populateHistory() { populateStats(); historicalDataElements.setPointerAndSizeArray = new FixedLengthElementArray(historicalDataElements.memoryRecycler, ((long)historicalDataElements.maxOrdinal + 1) * historicalDataElements.bitsPerFixedLengthSetPortion); historicalDataElements.elementArray = new FixedLengthElementArray(historicalDataElements.memoryRecycler, historicalDataElements.totalNumberOfBuckets * historicalDataElements.bitsPerElement); iter.reset(); int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { ordinalMapping.put(ordinal, nextOrdinal); copyRecord(ordinal); ordinal = iter.next(); } }
public void populateHistory() { populateStats(); historicalDataElements.listPointerArray = new FixedLengthElementArray(historicalDataElements.memoryRecycler, ((long)historicalDataElements.maxOrdinal + 1) * historicalDataElements.bitsPerListPointer); historicalDataElements.elementArray = new FixedLengthElementArray(historicalDataElements.memoryRecycler, historicalDataElements.totalNumberOfElements * historicalDataElements.bitsPerElement); iter.reset(); int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { ordinalMapping.put(ordinal, nextOrdinal); copyRecord(ordinal); ordinal = iter.next(); } }
private void populateStats() { iter.reset(); int removedEntryCount = 0; int maxSize = 0; long totalBucketCount = 0; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; int size = typeState.size(ordinal); if(size > maxSize) maxSize = size; totalBucketCount += HashCodes.hashTableSize(size); ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; historicalDataElements.bitsPerSetPointer = 64 - Long.numberOfLeadingZeros(totalBucketCount); historicalDataElements.bitsPerSetSizeValue = 64 - Long.numberOfLeadingZeros(maxSize); historicalDataElements.bitsPerFixedLengthSetPortion = historicalDataElements.bitsPerSetPointer + historicalDataElements.bitsPerSetSizeValue; historicalDataElements.bitsPerElement = stateEngineDataElements[0].bitsPerElement; historicalDataElements.emptyBucketValue = stateEngineDataElements[0].emptyBucketValue; historicalDataElements.totalNumberOfBuckets = totalBucketCount; ordinalMapping = new IntMap(removedEntryCount); }
public void populateHistory() { populateStats(); historicalDataElements.mapPointerAndSizeArray = new FixedLengthElementArray(historicalDataElements.memoryRecycler, ((long)historicalDataElements.maxOrdinal + 1) * historicalDataElements.bitsPerFixedLengthMapPortion); historicalDataElements.entryArray = new FixedLengthElementArray(historicalDataElements.memoryRecycler, historicalDataElements.totalNumberOfBuckets * historicalDataElements.bitsPerMapEntry); iter.reset(); int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { ordinalMapping.put(ordinal, nextOrdinal); copyRecord(ordinal); ordinal = iter.next(); } }
private void populateStats() { iter.reset(); int removedEntryCount = 0; int maxSize = 0; long totalBucketCount = 0; int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { removedEntryCount++; int size = typeState.size(ordinal); if(size > maxSize) maxSize = size; totalBucketCount += HashCodes.hashTableSize(size); ordinal = iter.next(); } historicalDataElements.maxOrdinal = removedEntryCount - 1; historicalDataElements.bitsPerMapPointer = 64 - Long.numberOfLeadingZeros(totalBucketCount); historicalDataElements.bitsPerMapSizeValue = 64 - Long.numberOfLeadingZeros(maxSize); historicalDataElements.bitsPerFixedLengthMapPortion = historicalDataElements.bitsPerMapPointer + historicalDataElements.bitsPerMapSizeValue; historicalDataElements.bitsPerKeyElement = stateEngineDataElements[0].bitsPerKeyElement; historicalDataElements.bitsPerValueElement = stateEngineDataElements[0].bitsPerValueElement; historicalDataElements.bitsPerMapEntry = stateEngineDataElements[0].bitsPerMapEntry; historicalDataElements.emptyBucketKeyValue = stateEngineDataElements[0].emptyBucketKeyValue; historicalDataElements.totalNumberOfBuckets = totalBucketCount; ordinalMapping = new IntMap(removedEntryCount); }
public void populateHistory() { populateStats(); historicalDataElements.fixedLengthData = new FixedLengthElementArray(historicalDataElements.memoryRecycler, (long)historicalDataElements.bitsPerRecord * (historicalDataElements.maxOrdinal + 1)); for(int i=0;i<historicalDataElements.schema.numFields();i++) { if(stateEngineDataElements[0].varLengthData[i] != null) { historicalDataElements.varLengthData[i] = new SegmentedByteArray(historicalDataElements.memoryRecycler); } } iter.reset(); int ordinal = iter.next(); while(ordinal != ORDINAL_NONE) { ordinalMapping.put(ordinal, nextOrdinal); copyRecord(ordinal); ordinal = iter.next(); } }