private HollowProducer createProducer(File tmpFolder, HollowObjectSchema... schemas) { HollowProducer producer = HollowProducer.withPublisher(new FakeBlobPublisher()) .withAnnouncer(new HollowFilesystemAnnouncer(tmpFolder.toPath())).build(); if (schemas != null && schemas.length > 0) { producer.initializeDataModel(schemas); } producer.addListener(new FakeProducerListener()); return producer; }
@Test public void testRollsBackStateEngineOnPublishFailure() throws Exception { HollowProducer producer = spy(createProducer(tmpFolder, schema)); Assert.assertEquals("Should have no populated ordinals", 0, producer.getWriteEngine().getTypeState("TestPojo").getPopulatedBitSet().cardinality()); doThrow(new RuntimeException("Publish failed")).when(producer).publish( any(ListenerSupport.Listeners.class), any(Long.class), any(Artifacts.class)); try { producer.runCycle(newState -> newState.add(new TestPojoV1(1, 1))); } catch (RuntimeException e) { // expected } Assert.assertEquals("Should still have no populated ordinals", 0, producer.getWriteEngine().getTypeState("TestPojo").getPopulatedBitSet().cardinality()); }
HollowProducer.ReadState hardRestore(long versionDesired, HollowConsumer.BlobRetriever blobRetriever) { return restore(versionDesired, blobRetriever, (restoreFrom, restoreTo) -> HollowWriteStateCreator.populateUsingReadEngine(restoreTo, restoreFrom)); }
/** * Registers {@code DuplicateDataDetectionValidator} validators with the given {@link HollowProducer producer} for * all object schema declared with a primary key. * <p> * This requires that the producer's data model has been initialized * (see {@link HollowProducer#initializeDataModel(Class[])} or a prior run cycle has implicitly initialized * the data model. * <p> * For each {@link HollowTypeWriteState write state} that has a {@link HollowObjectSchema object schema} * declared with a {@link PrimaryKey primary key} a {@code DuplicateDataDetectionValidator} validator * is instantiated, with the primary key type name, and registered with the given producer (if a * {@code DuplicateDataDetectionValidator} validator is not already registered for the same primary key type name). * * @param producer the producer * @apiNote This method registers a {@code DuplicateDataDetectionValidator} validator with only the primary key type * name and not, in addition, the primary key fields. This is to ensure, for the common case, duplicate listeners * are not registered by this method if listeners with the same type names were explicitly registered when * building the producer. * @see HollowProducer#initializeDataModel(Class[]) */ public static void addValidatorsForSchemaWithPrimaryKey(HollowProducer producer) { producer.getWriteEngine().getOrderedTypeStates().stream() .filter(ts -> ts.getSchema().getSchemaType() == SchemaType.OBJECT) .map(ts -> (HollowObjectSchema) ts.getSchema()) .filter(hos -> hos.getPrimaryKey() != null) .map(HollowObjectSchema::getPrimaryKey) .forEach(k -> producer.addListener(new DuplicateDataDetectionValidator(k.getType()))); } }
/** * Run a compaction cycle, will produce a data state with exactly the same data as currently, but * reorganized so that ordinal holes are filled. This may need to be run multiple times to arrive * at an optimal state. * * @param config specifies what criteria to use to determine whether a compaction is necessary * @return the version identifier of the produced state, or AnnouncementWatcher.NO_ANNOUNCEMENT_AVAILABLE if compaction was unnecessary. */ public long runCompactionCycle(HollowCompactor.CompactionConfig config) { if (config != null && readStates.hasCurrent()) { final HollowCompactor compactor = new HollowCompactor(getWriteEngine(), readStates.current().getStateEngine(), config); if (compactor.needsCompaction()) { return runCycle(newState -> compactor.compact()); } } return NO_ANNOUNCEMENT_AVAILABLE; }
@Test public void testNullMetricsCollector() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); long version = producer.runCycle(new HollowProducer.Populator() { public void populate(HollowProducer.WriteState state) throws Exception { state.add(Integer.valueOf(1)); } }); HollowConsumer consumer = HollowConsumer.withBlobRetriever(blobStore) .withMetricsCollector(null) .build(); consumer.triggerRefreshTo(version); }
long runCycle(ListenerSupport.Listeners listeners, Populator task, Status.StageWithStateBuilder cycleStatus, long toVersion) { HollowWriteStateEngine writeEngine = getWriteEngine(); try { publish(listeners, toVersion, artifacts); candidate = checkIntegrity(listeners, candidate, artifacts); validate(listeners, candidate.pending()); announce(listeners, candidate.pending()); } catch (Throwable th) { if (artifacts.hasReverseDelta()) { applyDelta(artifacts.reverseDelta, candidate.pending().getStateEngine()); readStates = candidate.rollback();
private void createHollowProducerAndRunCycle(final String typeName, boolean addPrimaryKeyValidator) { ValidatorListener dupeValidator = new DuplicateDataDetectionValidator(typeName); ValidatorListener countValidator = new RecordCountVarianceValidator(typeName, 3.0f); validationListener = new TestValidationStatusListener(); cycleAndValidationListener = new TestCycleAndValidationStatusListener(); Builder builder = HollowProducer.withPublisher(publisher).withAnnouncer(announcer) .withListener(validationListener) .withListener(cycleAndValidationListener) .withListener(countValidator); if (addPrimaryKeyValidator) { builder = builder.withListener(dupeValidator); } HollowProducer hollowProducer = builder.build(); if (typeName.equals("MovieWithPrimaryKey")) { hollowProducer.initializeDataModel(MovieWithPrimaryKey.class); } else { hollowProducer.initializeDataModel(MovieWithoutPrimaryKey.class); } hollowProducer.runCycle(newState -> { List<String> actors = Arrays.asList("Angelina Jolie", "Brad Pitt"); if (typeName.equals("MovieWithPrimaryKey")) { newState.add(new MovieWithPrimaryKey(123, "someTitle1", actors)); newState.add(new MovieWithPrimaryKey(123, "someTitle1", actors)); } else { newState.add(new MovieWithoutPrimaryKey(123, "someTitle1", actors)); newState.add(new MovieWithoutPrimaryKey(1233, "someTitle2", actors)); } }); }
private long runCycle(HollowProducer producer, final int cycleNumber) { return producer.runCycle(new Populator() { public void populate(WriteState state) throws Exception { state.add(Integer.valueOf(cycleNumber)); } }); }
@Test public void removeOrphanObjectsWithoutTypeInDelta() { HollowProducer producer = createInMemoryProducer(); producer.initializeDataModel(TypeC.class); producer.runCycle(new Populator() { public void populate(WriteState state) throws Exception { state.add(new TypeA(1, "one", 1)); } }); HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer); TypeD typeD2 = new TypeD(2, "two"); TypeC typeC2 = new TypeC(2, typeD2); incrementalProducer.addOrModify(typeC2); incrementalProducer.runCycle(); TypeD typeD3 = new TypeD(3, "three"); typeC2 = new TypeC(2, typeD3); //Modify typeC2 to point to a new TypeD object incrementalProducer.addOrModify(typeC2); //Cycle writes a snapshot long finalVersion = incrementalProducer.runCycle(); HollowConsumer consumer = HollowConsumer.withBlobRetriever(blobStore).build(); consumer.triggerRefreshTo(finalVersion); Collection<HollowObject> allHollowObjectsTypeD = getAllHollowObjects(consumer, "TypeD"); List<String> finalTypeDNames = new ArrayList<>(); for (HollowObject hollowObject : allHollowObjectsTypeD) { finalTypeDNames.add(((GenericHollowObject) hollowObject).getObject("value").toString()); } Assert.assertFalse(finalTypeDNames.contains("two")); }
@Test public void metricsWhenPublishingFails() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); try { producer.runCycle(new HollowProducer.Populator() { public void populate(HollowProducer.WriteState state) throws Exception { state.add(null); } }); } catch (Exception ignored){ } HollowProducerMetrics hollowProducerMetrics = producer.getMetrics(); Assert.assertEquals(hollowProducerMetrics.getCyclesSucceeded(), 0); Assert.assertEquals(hollowProducerMetrics.getCyclesCompleted(), 1); Assert.assertEquals(hollowProducerMetrics.getCycleFailed(), 1); } }
@Test public void producerRestoresAndProducesDelta() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); long v1 = runCycle(producer, 1); HollowProducer redeployedProducer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); redeployedProducer.initializeDataModel(Integer.class); redeployedProducer.restore(v1, blobStore); long v2 = runCycle(producer, 2); Assert.assertNotNull(blobStore.retrieveDeltaBlob(v1)); Assert.assertEquals(v2, blobStore.retrieveDeltaBlob(v1).getToVersion()); }
@Test public void testNotPrimaryProducerVersion() { BasicSingleProducerEnforcer enforcer = new BasicSingleProducerEnforcer(); HollowProducer producer = HollowProducer.withPublisher(new FakeBlobPublisher()) .withSingleProducerEnforcer(enforcer) .withAnnouncer(new HollowFilesystemAnnouncer(tmpFolder.toPath())) .build(); producer.addListener(new FakeProducerListener()); long v1 = producer.runCycle(ws -> { ws.add(1); }); enforcer.disable(); // Run cycle as not the primary producer long v2 = producer.runCycle(ws -> { ws.add(1); }); // Run cycle as the primary producer enforcer.enable(); long v3 = producer.runCycle(ws -> { ws.add(2); }); Assert.assertEquals(v1, v2); Assert.assertTrue(v3 > v2); }
@Test public void producerCompacts() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); producer.runCycle(state -> { for (int i = 0; i < 10000; i++) { state.add(i); long v2 = producer.runCycle(state -> { for (int i = 10000; i < 20000; i++) { state.add(i); long v3 = producer.runCompactionCycle(new CompactionConfig(0, 20));
backingProducer.initializeDataModel(TypeA.class, TypeB.class); backingProducer.restore(nextVersion, blobStore);
/** * Initializes the producer data model for the given schemas. * <p> * Data model initialization is required prior to {@link #restore(long, HollowConsumer.BlobRetriever) restoring} * the producer. * This ensures that restoration can correctly compare the producer's current data model * with the data model of the restored data state and manage any differences in those models * (such as not restoring state for any types in the restoring data model not present in the * producer's current data model). * <p> * After initialization a data model initialization event will be emitted * to all registered data model initialization * {@link com.netflix.hollow.api.producer.listener.DataModelInitializationListener listeners}. * * @param schemas the data model classes * @throws IllegalArgumentException if {@code schemas} is empty * @see #restore(long, HollowConsumer.BlobRetriever) */ public void initializeDataModel(HollowSchema... schemas) { Objects.requireNonNull(schemas); if (schemas.length == 0) { throw new IllegalArgumentException("classes is empty"); } long start = currentTimeMillis(); HollowWriteStateCreator.populateStateEngineWithTypeWriteStates(getWriteEngine(), Arrays.asList(schemas)); listeners.listeners().fireProducerInit(currentTimeMillis() - start); isInitialized = true; }
/** * Initializes the data model and restores from existing state. */ public void restoreFromLastState() { producer.initializeDataModel(dataModel); long latestAnnouncedVersion = announcementWatcher.getLatestVersion(); if (latestAnnouncedVersion == HollowFilesystemAnnouncementWatcher.NO_ANNOUNCEMENT_AVAILABLE || latestAnnouncedVersion < 0) { return; } restore(latestAnnouncedVersion, blobRetriever); }
ReadStateHelper result = readStates; HollowReadStateEngine pending = readStates.pending().getStateEngine(); readSnapshot(artifacts.snapshot, pending); applyDelta(artifacts.delta, current); HollowChecksum forwardChecksum = HollowChecksum.forStateEngineWithCommonSchemas(current, pending); applyDelta(artifacts.reverseDelta, pending); HollowChecksum reverseChecksum = HollowChecksum.forStateEngineWithCommonSchemas(pending, current);
public HollowProducer build() { checkArguments(); return new HollowProducer(this); } }
@Test public void publishAndLoadASnapshot() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); /// Showing verbose version of `runCycle(producer, 1);` long version = producer.runCycle(state -> state.add(1)); HollowConsumer consumer = HollowConsumer.withBlobRetriever(blobStore).build(); consumer.triggerRefreshTo(version); Assert.assertEquals(version, consumer.getCurrentVersionId()); }