/** * Initializes the data model and restores from existing state. */ public void restoreFromLastState() { producer.initializeDataModel(dataModel); long latestAnnouncedVersion = announcementWatcher.getLatestVersion(); if (latestAnnouncedVersion == HollowFilesystemAnnouncementWatcher.NO_ANNOUNCEMENT_AVAILABLE || latestAnnouncedVersion < 0) { return; } restore(latestAnnouncedVersion, blobRetriever); }
private HollowProducer createProducer(File tmpFolder, HollowObjectSchema... schemas) { HollowProducer producer = HollowProducer.withPublisher(new FakeBlobPublisher()) .withAnnouncer(new HollowFilesystemAnnouncer(tmpFolder.toPath())).build(); if (schemas != null && schemas.length > 0) { producer.initializeDataModel(schemas); } producer.addListener(new FakeProducerListener()); return producer; }
private void createHollowProducerAndRunCycle(final String typeName, boolean addPrimaryKeyValidator) { ValidatorListener dupeValidator = new DuplicateDataDetectionValidator(typeName); ValidatorListener countValidator = new RecordCountVarianceValidator(typeName, 3.0f); validationListener = new TestValidationStatusListener(); cycleAndValidationListener = new TestCycleAndValidationStatusListener(); Builder builder = HollowProducer.withPublisher(publisher).withAnnouncer(announcer) .withListener(validationListener) .withListener(cycleAndValidationListener) .withListener(countValidator); if (addPrimaryKeyValidator) { builder = builder.withListener(dupeValidator); } HollowProducer hollowProducer = builder.build(); if (typeName.equals("MovieWithPrimaryKey")) { hollowProducer.initializeDataModel(MovieWithPrimaryKey.class); } else { hollowProducer.initializeDataModel(MovieWithoutPrimaryKey.class); } hollowProducer.runCycle(newState -> { List<String> actors = Arrays.asList("Angelina Jolie", "Brad Pitt"); if (typeName.equals("MovieWithPrimaryKey")) { newState.add(new MovieWithPrimaryKey(123, "someTitle1", actors)); newState.add(new MovieWithPrimaryKey(123, "someTitle1", actors)); } else { newState.add(new MovieWithoutPrimaryKey(123, "someTitle1", actors)); newState.add(new MovieWithoutPrimaryKey(1233, "someTitle2", actors)); } }); }
@Test public void removeOrphanObjectsWithoutTypeInDelta() { HollowProducer producer = createInMemoryProducer(); producer.initializeDataModel(TypeC.class); producer.runCycle(new Populator() { public void populate(WriteState state) throws Exception { state.add(new TypeA(1, "one", 1)); } }); HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer); TypeD typeD2 = new TypeD(2, "two"); TypeC typeC2 = new TypeC(2, typeD2); incrementalProducer.addOrModify(typeC2); incrementalProducer.runCycle(); TypeD typeD3 = new TypeD(3, "three"); typeC2 = new TypeC(2, typeD3); //Modify typeC2 to point to a new TypeD object incrementalProducer.addOrModify(typeC2); //Cycle writes a snapshot long finalVersion = incrementalProducer.runCycle(); HollowConsumer consumer = HollowConsumer.withBlobRetriever(blobStore).build(); consumer.triggerRefreshTo(finalVersion); Collection<HollowObject> allHollowObjectsTypeD = getAllHollowObjects(consumer, "TypeD"); List<String> finalTypeDNames = new ArrayList<>(); for (HollowObject hollowObject : allHollowObjectsTypeD) { finalTypeDNames.add(((GenericHollowObject) hollowObject).getObject("value").toString()); } Assert.assertFalse(finalTypeDNames.contains("two")); }
void duplicateDetectionFailureTest(DuplicateDataDetectionValidator v, boolean auto) { HollowProducer.Builder<?> b = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()); if (v != null) { b.withListener(v); } HollowProducer producer = b.build(); if (auto) { producer.initializeDataModel(TypeWithPrimaryKey.class); DuplicateDataDetectionValidator.addValidatorsForSchemaWithPrimaryKey(producer); } try { //runCycle(producer, 1); producer.runCycle(newState -> { newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "klsdjfla;sdjkf")); newState.add(new TypeWithPrimaryKey(1, "Angelina Jolie", "as;dlkfjasd;l")); newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "as;dlkfjasd;l")); }); Assert.fail(); } catch (ValidationStatusException expected) { Assert.assertEquals(1, expected.getValidationStatus().getResults().size()); Assert.assertTrue(expected.getValidationStatus().getResults().get(0).getMessage() .startsWith("Duplicate keys found for type TypeWithPrimaryKey")); } }
@Test public void producerRestoresAndProducesDelta() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); long v1 = runCycle(producer, 1); HollowProducer redeployedProducer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); redeployedProducer.initializeDataModel(Integer.class); redeployedProducer.restore(v1, blobStore); long v2 = runCycle(producer, 2); Assert.assertNotNull(blobStore.retrieveDeltaBlob(v1)); Assert.assertEquals(v2, blobStore.retrieveDeltaBlob(v1).getToVersion()); }
backingProducer.initializeDataModel(TypeA.class, TypeB.class);
@Test public void flatRecordsCanBeDumpedToStateEngineWithIdenticalSchemas() throws IOException { producer.initializeDataModel(TypeA.class, TypeC.class);
typeCSchema.addField("c3", FieldType.FLOAT); producer.initializeDataModel(typeASchema, typeBSchema, typeCSchema);
backingProducer.initializeDataModel(TypeA.class, TypeB.class); backingProducer.restore(nextVersion, blobStore);