/** * Initializes the producer data model for the given schemas. * <p> * Data model initialization is required prior to {@link #restore(long, HollowConsumer.BlobRetriever) restoring} * the producer. * This ensures that restoration can correctly compare the producer's current data model * with the data model of the restored data state and manage any differences in those models * (such as not restoring state for any types in the restoring data model not present in the * producer's current data model). * <p> * After initialization a data model initialization event will be emitted * to all registered data model initialization * {@link com.netflix.hollow.api.producer.listener.DataModelInitializationListener listeners}. * * @param schemas the data model classes * @throws IllegalArgumentException if {@code schemas} is empty * @see #restore(long, HollowConsumer.BlobRetriever) */ public void initializeDataModel(HollowSchema... schemas) { Objects.requireNonNull(schemas); if (schemas.length == 0) { throw new IllegalArgumentException("classes is empty"); } long start = currentTimeMillis(); HollowWriteStateCreator.populateStateEngineWithTypeWriteStates(getWriteEngine(), Arrays.asList(schemas)); listeners.listeners().fireProducerInit(currentTimeMillis() - start); isInitialized = true; }
/** * Registers {@code DuplicateDataDetectionValidator} validators with the given {@link HollowProducer producer} for * all object schema declared with a primary key. * <p> * This requires that the producer's data model has been initialized * (see {@link HollowProducer#initializeDataModel(Class[])} or a prior run cycle has implicitly initialized * the data model. * <p> * For each {@link HollowTypeWriteState write state} that has a {@link HollowObjectSchema object schema} * declared with a {@link PrimaryKey primary key} a {@code DuplicateDataDetectionValidator} validator * is instantiated, with the primary key type name, and registered with the given producer (if a * {@code DuplicateDataDetectionValidator} validator is not already registered for the same primary key type name). * * @param producer the producer * @apiNote This method registers a {@code DuplicateDataDetectionValidator} validator with only the primary key type * name and not, in addition, the primary key fields. This is to ensure, for the common case, duplicate listeners * are not registered by this method if listeners with the same type names were explicitly registered when * building the producer. * @see HollowProducer#initializeDataModel(Class[]) */ public static void addValidatorsForSchemaWithPrimaryKey(HollowProducer producer) { producer.getWriteEngine().getOrderedTypeStates().stream() .filter(ts -> ts.getSchema().getSchemaType() == SchemaType.OBJECT) .map(ts -> (HollowObjectSchema) ts.getSchema()) .filter(hos -> hos.getPrimaryKey() != null) .map(HollowObjectSchema::getPrimaryKey) .forEach(k -> producer.addListener(new DuplicateDataDetectionValidator(k.getType()))); } }
@Test public void testRollsBackStateEngineOnPublishFailure() throws Exception { HollowProducer producer = spy(createProducer(tmpFolder, schema)); Assert.assertEquals("Should have no populated ordinals", 0, producer.getWriteEngine().getTypeState("TestPojo").getPopulatedBitSet().cardinality()); doThrow(new RuntimeException("Publish failed")).when(producer).publish( any(ListenerSupport.Listeners.class), any(Long.class), any(Artifacts.class)); try { producer.runCycle(newState -> newState.add(new TestPojoV1(1, 1))); } catch (RuntimeException e) { // expected } Assert.assertEquals("Should still have no populated ordinals", 0, producer.getWriteEngine().getTypeState("TestPojo").getPopulatedBitSet().cardinality()); }
/** * Run a compaction cycle, will produce a data state with exactly the same data as currently, but * reorganized so that ordinal holes are filled. This may need to be run multiple times to arrive * at an optimal state. * * @param config specifies what criteria to use to determine whether a compaction is necessary * @return the version identifier of the produced state, or AnnouncementWatcher.NO_ANNOUNCEMENT_AVAILABLE if compaction was unnecessary. */ public long runCompactionCycle(HollowCompactor.CompactionConfig config) { if (config != null && readStates.hasCurrent()) { final HollowCompactor compactor = new HollowCompactor(getWriteEngine(), readStates.current().getStateEngine(), config); if (compactor.needsCompaction()) { return runCycle(newState -> compactor.compact()); } } return NO_ANNOUNCEMENT_AVAILABLE; }
private void stageBlob(long toVersion, Artifacts artifacts, Blob.Type blobType) throws IOException { HollowBlobWriter writer = new HollowBlobWriter(getWriteEngine()); switch (blobType) { case SNAPSHOT: artifacts.snapshot = blobStager.openSnapshot(toVersion); artifacts.snapshot.write(writer); break; case DELTA: artifacts.delta = blobStager.openDelta(readStates.current().getVersion(), toVersion); artifacts.delta.write(writer); break; case REVERSE_DELTA: artifacts.reverseDelta = blobStager.openReverseDelta(toVersion, readStates.current().getVersion()); artifacts.reverseDelta.write(writer); break; default: throw new IllegalStateException("unknown type, type=" + blobType); } }
long runCycle(ListenerSupport.Listeners listeners, Populator task, Status.StageWithStateBuilder cycleStatus, long toVersion) { HollowWriteStateEngine writeEngine = getWriteEngine(); try {