@Override public ValidationResult onValidate(ReadState readState) { ValidationResult.ValidationResultBuilder vrb = ValidationResult.from(this); vrb.detail(DATA_TYPE_NAME, dataTypeName); PrimaryKey primaryKey; if (fieldPathNames == null) { HollowSchema schema = readState.getStateEngine().getSchema(dataTypeName); if (schema.getSchemaType() != SchemaType.OBJECT) { return vrb.failed(String.format(NOT_AN_OBJECT_ERROR_MSGR_FORMAT, dataTypeName)); } HollowObjectSchema oSchema = (HollowObjectSchema) schema; primaryKey = oSchema.getPrimaryKey(); if (primaryKey == null) { return vrb.failed(String.format(NO_PRIMARY_KEY_ERRRO_MSG_FORMAT, dataTypeName)); } } else { primaryKey = new PrimaryKey(dataTypeName, fieldPathNames); } String fieldPaths = Arrays.toString(primaryKey.getFieldPaths()); vrb.detail(FIELD_PATH_NAME, fieldPaths); Collection<Object[]> duplicateKeys = getDuplicateKeys(readState.getStateEngine(), primaryKey); if (!duplicateKeys.isEmpty()) { String message = String.format(DUPLICATE_KEYS_FOUND_ERRRO_MSG_FORMAT, dataTypeName, fieldPaths, duplicateKeysToString(duplicateKeys)); return vrb.failed(message); } return vrb.passed(); }
@Test public void fireProducerRestoreStartDontStopWhenOneFails() { long version = 31337; HollowProducer.ReadState readState = Mockito.mock(HollowProducer.ReadState.class); Mockito.when(readState.getVersion()).thenReturn(version); Mockito.doThrow(RuntimeException.class).when(listener).onProducerRestoreStart(version); Status.RestoreStageBuilder b = new Status.RestoreStageBuilder(); listenerSupport.listeners().fireProducerRestoreComplete(b); ArgumentCaptor<Status> status = ArgumentCaptor.forClass( Status.class); ArgumentCaptor<Long> desired = ArgumentCaptor.forClass( long.class); ArgumentCaptor<Long> reached = ArgumentCaptor.forClass( long.class); ArgumentCaptor<Duration> elapsed = ArgumentCaptor.forClass( Duration.class); Mockito.verify(listener).onProducerRestoreComplete(status.capture(), desired.capture(), reached.capture(), elapsed.capture()); Assert.assertNotNull(status.getValue()); Assert.assertNotNull(elapsed.getValue()); }
} catch (Throwable th) { if (artifacts.hasReverseDelta()) { applyDelta(artifacts.reverseDelta, candidate.pending().getStateEngine()); readStates = candidate.rollback();
try { ReadStateHelper result = readStates; HollowReadStateEngine pending = readStates.pending().getStateEngine(); readSnapshot(artifacts.snapshot, pending); HollowReadStateEngine current = readStates.current().getStateEngine();
.detail("Typename", typeName); HollowTypeReadState typeState = readState.getStateEngine().getTypeState(typeName); if (typeState == null) { return vrb.failed("Cannot execute ObjectModificationValidator on missing type " + typeName); A hollowApi = apiFunction.apply(readState.getStateEngine()); HollowObjectTypeReadState objectTypeState = (HollowObjectTypeReadState) typeState; PrimaryKey key = objectTypeState.getSchema().getPrimaryKey(); HollowPrimaryKeyIndex index = new HollowPrimaryKeyIndex(readState.getStateEngine(), key); int fromOrdinal = removedAndModified.nextSetBit(0); while (fromOrdinal != HollowConstants.ORDINAL_NONE) {
restoreAction.restore(readStates.current().getStateEngine(), newObjectMapper.getStateEngine()); status.versions(versionDesired, readState.getVersion()) .success();
@Override public ValidationResult onValidate(ReadState readState) { ValidationResult.ValidationResultBuilder vrb = ValidationResult.from(this); vrb.detail(ALLOWABLE_VARIANCE_PERCENT_NAME, allowableVariancePercent) .detail(DATA_TYPE_NAME, typeName); HollowTypeReadState typeState = readState.getStateEngine().getTypeState(typeName); int latestCardinality = typeState.getPopulatedOrdinals().cardinality(); int previousCardinality = typeState.getPreviousOrdinals().cardinality(); vrb.detail(LATEST_CARDINALITY_NAME, latestCardinality) .detail(PREVIOUS_CARDINALITY_NAME, previousCardinality); if (previousCardinality == 0) { return vrb.detail("skipped", Boolean.TRUE). passed(String.format(ZERO_PREVIOUS_COUNT_WARN_MSG_FORMAT, typeName)); } float actualChangePercent = getChangePercent(latestCardinality, previousCardinality); vrb.detail(ACTUAL_CHANGE_PERCENT_NAME, actualChangePercent); if (Float.compare(actualChangePercent, allowableVariancePercent) > 0) { String message = String.format(FAILED_RECORD_COUNT_VALIDATION, typeName, actualChangePercent, allowableVariancePercent); return vrb.failed(message); } return vrb.passed(); }
private void restoreAndAssert(HollowProducer producer, long version, int size, int valueMultiplier, int valueFieldCount) { ReadState readState = producer.restore(version, blobRetriever); Assert.assertNotNull(lastRestoreStatus); Assert.assertEquals(Status.SUCCESS, lastRestoreStatus.getStatus()); Assert.assertEquals("Version should be the same", version, lastRestoreStatus.getDesiredVersion()); HollowObjectTypeReadState typeState = (HollowObjectTypeReadState) readState.getStateEngine().getTypeState("TestPojo"); BitSet populatedOrdinals = typeState.getPopulatedOrdinals(); Assert.assertEquals(size, populatedOrdinals.cardinality()); int ordinal = populatedOrdinals.nextSetBit(0); while (ordinal != -1) { GenericHollowObject obj = new GenericHollowObject(new HollowObjectGenericDelegate(typeState), ordinal); System.out.println("ordinal=" + ordinal + obj); int id = obj.getInt("id"); for (int i = 0; i < valueFieldCount; i++) { String valueFN = "v" + (i + 1); int value = id * valueMultiplier; Assert.assertEquals(valueFN, value, obj.getInt(valueFN)); } ordinal = populatedOrdinals.nextSetBit(ordinal + 1); } System.out.println("Asserted Correctness of version:" + version + "\n\n"); }
/** * Updates the producer metrics: * cycles completed, version and type's footprint and ordinals. * @param status the status * @param readState the read state * @param version the version */ public void updateCycleMetrics(Status status, HollowProducer.ReadState readState, long version) { cyclesCompleted++; if(status.getType() == Status.StatusType.FAIL) { cycleFailed++; return; } cyclesSucceeded++; if(readState != null) { HollowReadStateEngine hollowReadStateEngine = readState.getStateEngine(); super.update(hollowReadStateEngine, version); } else { super.update(version); } }
private void stageBlob(long toVersion, Artifacts artifacts, Blob.Type blobType) throws IOException { HollowBlobWriter writer = new HollowBlobWriter(getWriteEngine()); switch (blobType) { case SNAPSHOT: artifacts.snapshot = blobStager.openSnapshot(toVersion); artifacts.snapshot.write(writer); break; case DELTA: artifacts.delta = blobStager.openDelta(readStates.current().getVersion(), toVersion); artifacts.delta.write(writer); break; case REVERSE_DELTA: artifacts.reverseDelta = blobStager.openReverseDelta(toVersion, readStates.current().getVersion()); artifacts.reverseDelta.write(writer); break; default: throw new IllegalStateException("unknown type, type=" + blobType); } }
/** * Run a compaction cycle, will produce a data state with exactly the same data as currently, but * reorganized so that ordinal holes are filled. This may need to be run multiple times to arrive * at an optimal state. * * @param config specifies what criteria to use to determine whether a compaction is necessary * @return the version identifier of the produced state, or AnnouncementWatcher.NO_ANNOUNCEMENT_AVAILABLE if compaction was unnecessary. */ public long runCompactionCycle(HollowCompactor.CompactionConfig config) { if (config != null && readStates.hasCurrent()) { final HollowCompactor compactor = new HollowCompactor(getWriteEngine(), readStates.current().getStateEngine(), config); if (compactor.needsCompaction()) { return runCycle(newState -> compactor.compact()); } } return NO_ANNOUNCEMENT_AVAILABLE; }
private void announce(ListenerSupport.Listeners listeners, HollowProducer.ReadState readState) { if (announcer != null) { Status.StageWithStateBuilder status = listeners.fireAnnouncementStart(readState); try { announcer.announce(readState.getVersion()); status.success(); } catch (Throwable th) { status.fail(th); throw th; } finally { listeners.fireAnnouncementComplete(status); } } }
StageWithStateBuilder readState(HollowProducer.ReadState readState) { this.readState = readState; return version(readState.getVersion()); }
long pendingVersion() { return pending != null ? pending.getVersion() : HollowConstants.VERSION_NONE; } }
ReadStateHelper rollback() { if(pending == null) throw new IllegalStateException(); return new ReadStateHelper(newReadState(current.getVersion(), pending.getStateEngine()), null); }
private Map<String, BitSet> markRecordsToRemove(HollowProducer.ReadState priorState, Collection<String> types) { HollowReadStateEngine priorStateEngine = priorState.getStateEngine(); Map<String, BitSet> recordsToRemove = new HashMap<>(); for(String type : types) { recordsToRemove.put(type, markTypeRecordsToRemove(priorStateEngine, type)); } TransitiveSetTraverser.addTransitiveMatches(priorStateEngine, recordsToRemove); TransitiveSetTraverser.removeReferencedOutsideClosure(priorStateEngine, recordsToRemove); return recordsToRemove; }
Status.StageWithStateBuilder fireIntegrityCheckStart(HollowProducer.ReadState readState) { long version = readState.getVersion(); fire(IntegrityCheckListener.class, l -> l.onIntegrityCheckStart(version)); return new Status.StageWithStateBuilder().readState(readState); }
@Test public void fireNewDeltaChainDontStopWhenOneFails() { long version = 31337; HollowProducer.ReadState readState = Mockito.mock(HollowProducer.ReadState.class); Mockito.when(readState.getVersion()).thenReturn(version); Mockito.doThrow(RuntimeException.class).when(listener).onNewDeltaChain(version); listenerSupport.listeners().fireNewDeltaChain(version); Mockito.verify(listener).onNewDeltaChain(version); }
@Test public void testFireValidationStartDontStopWhenOneFails2() { long version = 31337; HollowProducer.ReadState readState = Mockito.mock(HollowProducer.ReadState.class); Mockito.when(readState.getVersion()).thenReturn(version); Mockito.doThrow(RuntimeException.class).when(validationStatusListener).onValidationStatusStart(version); listenerSupport.listeners().fireValidationStart(readState); Mockito.verify(listener).onValidationStart(version); Mockito.verify(validationStatusListener).onValidationStatusStart(version); Mockito.verify(producerAndValidationStatusListener).onValidationStart(version); }
/** * Swap underlying state engines between current and pending while keeping the versions consistent; * used after delta integrity checks have altered the underlying state engines. * * @return */ ReadStateHelper swap() { return new ReadStateHelper(newReadState(current.getVersion(), pending.getStateEngine()), newReadState(pending.getVersion(), current.getStateEngine())); }