public HollowIncrementalProducer build() { checkArguments(); return new HollowIncrementalProducer(producer, threadsPerCpu, announcementWatcher, blobRetriever, listeners, dataModel); } }
public void addOrModify(Collection<Object> objList) { for(Object obj : objList) { addOrModify(obj); } }
public void delete(Object obj) { RecordPrimaryKey pk = extractRecordPrimaryKey(obj); delete(pk); }
public void discard(Object obj) { RecordPrimaryKey pk = extractRecordPrimaryKey(obj); discard(pk); }
@Test public void clearMutations() { HollowProducer producer = createInMemoryProducer(); /// initialize the data -- classic producer creates the first state in the delta chain. initializeData(producer); /// now we'll be incrementally updating the state by mutating individual records HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer); incrementalProducer.addOrModify(new TypeA(1, "one", 100)); incrementalProducer.addOrModify(new TypeA(2, "two", 2)); incrementalProducer.addOrModify(new TypeA(3, "three", 300)); incrementalProducer.addOrModify(new TypeA(3, "three", 3)); incrementalProducer.addOrModify(new TypeA(4, "five", 6)); incrementalProducer.delete(new TypeA(5, "five", 5)); incrementalProducer.delete(new TypeB(2, "3")); incrementalProducer.addOrModify(new TypeB(5, "5")); incrementalProducer.addOrModify(new TypeB(5, "6")); incrementalProducer.delete(new RecordPrimaryKey("TypeB", new Object[]{3})); Assert.assertTrue(incrementalProducer.hasChanges()); /// .runCycle() flushes the changes to a new data state. incrementalProducer.runCycle(); Assert.assertFalse(incrementalProducer.hasChanges()); }
HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer); incrementalProducer.addOrModify(new TypeA(2, "two", 100)); incrementalProducer.runCycle(); incrementalProducer.addOrModify(new TypeA(3, "three", 1000)); incrementalProducer.runCycle(); incrementalProducer.addOrModify(new TypeA(4, "two", 100)); incrementalProducer.runCycle(); incrementalProducer.addOrModify(new TypeA(5, "three", 1000)); incrementalProducer.runCycle(); Assert.assertEquals(5, files.length); incrementalProducer.addOrModify(new TypeA(6, "three", 1000)); incrementalProducer.runCycle(); incrementalProducer.addOrModify(new TypeA(7, "three", 1000)); incrementalProducer.runCycle();
@Test public void fireFailureListener() { HollowProducer producer = createInMemoryProducer(); /// initialize the data -- classic producer creates the first state in the delta chain. initializeData(producer); FakeIncrementalCycleListener listener = new FakeIncrementalCycleListener(); HollowProducer fakeHollowProducer = FakeHollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .withVersionMinter(new TestVersionMinter()) .build(); HollowProducer fakeHollowProducerSpy = Mockito.spy(fakeHollowProducer); /// now we'll be incrementally updating the state by mutating individual records HollowIncrementalProducer incrementalProducer = HollowIncrementalProducer .withProducer(fakeHollowProducerSpy) .withListener(listener) .build(); incrementalProducer.addOrModify(new TypeA(1, "one", 100)); incrementalProducer.addOrModify(new TypeA(2, "two", 2)); incrementalProducer.addOrModify(new TypeA(3, "three", 300)); incrementalProducer.addOrModify(new TypeA(3, "three", 3)); incrementalProducer.addOrModify(new TypeA(4, "five", 6)); incrementalProducer.delete(new TypeA(5, "five", 5)); Mockito.doThrow(new RuntimeException("oops")).when(fakeHollowProducerSpy).runCycle(any(HollowProducer.Populator.class)); long nextVersion = incrementalProducer.runCycle(); Assert.assertEquals(nextVersion, listener.getVersion()); Assert.assertEquals(IncrementalCycleListener.Status.FAIL, listener.getStatus()); Assert.assertEquals(4L, listener.getRecordsAddedOrModified()); Assert.assertEquals(1L, listener.getRecordsRemoved()); Assert.assertNotNull(listener.getCause()); }
.withProducer(producer) .withListener(listener) .build(); incrementalProducer.addAllCycleMetadata(cycleMetadata); Assert.assertTrue(incrementalProducer.hasMetadata()); incrementalProducer.addCycleMetadata("key2", "baz"); incrementalProducer.addCycleMetadata("key3", "baz"); incrementalProducer.addOrModify(new TypeA(1, "one", 100)); incrementalProducer.removeFromCycleMetadata("key2"); long nextVersion = incrementalProducer.runCycle(); Assert.assertEquals(nextVersion, listener.getVersion()); Assert.assertEquals(IncrementalCycleListener.Status.SUCCESS, listener.getStatus()); Assert.assertFalse(incrementalProducer.hasMetadata()); incrementalProducer.addOrModify(new TypeA(1, "one", 1000)); incrementalProducer.runCycle(); Assert.assertEquals(new HashMap<String, Object>(), listener.getCycleMetadata());
HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(backingProducer); incrementalProducer.restore(originalVersion, blobStore); incrementalProducer.addOrModify(new TypeA(1, "one", 2)); incrementalProducer.addOrModify(new TypeA(2, "two", 2)); incrementalProducer.addOrModify(new TypeB(3, "three")); long version = incrementalProducer.runCycle();
HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer); ); incrementalProducer.addOrModifyInParallel(addOrModifyList); incrementalProducer.deleteInParallel(deleteList); long nextVersion = incrementalProducer.runCycle(); incrementalProducer.addOrModify(new TypeA(1, "one", 1000)); long finalVersion = incrementalProducer.runCycle();
HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer); incrementalProducer.addIfAbsent(new TypeA(100, "one hundred", 9999)); // new incrementalProducer.addIfAbsent(new TypeA(101, "one hundred and one", 9998)); // new incrementalProducer.addIfAbsent(new TypeA(1, "one", 9997)); // exists in prior state incrementalProducer.addIfAbsent(new TypeA(2, "two", 9996)); // exists in prior state incrementalProducer.addOrModify(new TypeA(102, "one hundred and two", 9995)); // new incrementalProducer.addIfAbsent(new TypeA(102, "one hundred and two", 9994)); // new, but already added incrementalProducer.addIfAbsent(new TypeA(103, "one hundred and three", 9993)); // new incrementalProducer.addOrModify(new TypeA(103, "one hundred and three", 9992)); // overwrites prior call to addIfAbsent long version = incrementalProducer.runCycle();
@Test public void fireFailureListenerWithMetadata() { HollowProducer producer = createInMemoryProducer(); /// initialize the data -- classic producer creates the first state in the delta chain. initializeData(producer); FakeIncrementalCycleListener listener = new FakeIncrementalCycleListener(); HollowProducer fakeHollowProducer = FakeHollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .withVersionMinter(new TestVersionMinter()) .build(); HollowProducer fakeHollowProducerSpy = Mockito.spy(fakeHollowProducer); /// now we'll be incrementally updating the state by mutating individual records HollowIncrementalProducer incrementalProducer = HollowIncrementalProducer .withProducer(fakeHollowProducerSpy) .withListener(listener) .build(); HashMap<String, Object> cycleMetadata = new HashMap<>(); cycleMetadata.put("foo", "bar"); incrementalProducer.addAllCycleMetadata(cycleMetadata); Assert.assertTrue(incrementalProducer.hasMetadata()); incrementalProducer.addOrModify(new TypeA(1, "one", 100)); Mockito.doThrow(new RuntimeException("oops")).when(fakeHollowProducerSpy).runCycle(any(HollowProducer.Populator.class)); incrementalProducer.runCycle(); Assert.assertEquals(IncrementalCycleListener.Status.FAIL, listener.getStatus()); Assert.assertEquals(cycleMetadata, listener.getCycleMetadata()); Assert.assertFalse(incrementalProducer.hasMetadata()); }
@Test public void removeAndPublishUsingMultithreading() { // run within a loop to increase the likelihood of a race condition to occur for (int iterationCounter = 0; iterationCounter < ITERATIONS; ++iterationCounter) { HollowProducer producer = createInMemoryProducer(); /// initialize the data -- classic producer creates the first state in the delta chain. initializeData(producer); /// now we'll be incrementally updating the state by mutating individual records HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer, THREADS); int[] notModifiedElementIds = IntStream.range(0, ELEMENTS / 2).toArray(); int[] deletedElementIds = IntStream.range(ELEMENTS / 2, ELEMENTS).toArray(); incrementalProducer.delete(Arrays.stream(deletedElementIds).mapToObj(i -> new SimpleType(i, i)).collect(Collectors.toList())); /// .runCycle() flushes the changes to a new data state. long versionAfterDelete = incrementalProducer.runCycle(); /// now we read the changes and assert HollowPrimaryKeyIndex idx = createPrimaryKeyIndex(versionAfterDelete); Assert.assertTrue(Arrays.stream(notModifiedElementIds) .boxed() .map(elementId -> getHollowObject(idx, elementId)) .allMatch(obj -> obj.getInt("value") == obj.getInt("id"))); Assert.assertTrue(Arrays.stream(deletedElementIds) .boxed() .map(elementId -> getOrdinal(idx, elementId)) .allMatch(ordinal -> ordinal == -1)); } }
@Test public void addsListenersAfterInitialization() { HollowProducer producer = createInMemoryProducer(); initializeData(producer); FakeIncrementalCycleListener listener = new FakeIncrementalCycleListener(); HollowIncrementalProducer incrementalProducer = HollowIncrementalProducer .withProducer(producer) .build(); //Adding the listener after initialization incrementalProducer.addListener(listener); incrementalProducer.addOrModify(new TypeA(1, "one", 100)); /// .runCycle() flushes the changes to a new data state. long nextVersion = incrementalProducer.runCycle(); Assert.assertEquals(nextVersion, listener.getVersion()); Assert.assertEquals(IncrementalCycleListener.Status.SUCCESS, listener.getStatus()); Assert.assertEquals(1L, listener.getRecordsAddedOrModified()); Assert.assertNull(listener.getCause()); }
.build(); HollowIncrementalProducer incrementalProducer = HollowIncrementalProducer.withProducer(backingProducer) .withBlobRetriever(blobStore) .withAnnouncementWatcher(new FakeAnnouncementWatcher(originalVersion)) incrementalProducer.restoreFromLastState(); incrementalProducer.addOrModify(new TypeA(1, "one", 2)); incrementalProducer.addOrModify(new TypeA(2, "two", 2)); incrementalProducer.addOrModify(new TypeB(3, "three")); long version = incrementalProducer.runCycle();
public void discard(Collection<Object> objList) { for(Object obj : objList) { discard(obj); } }
public void delete(Collection<Object> objList) { for(Object obj : objList) { delete(obj); } }
/** * Initializes the data model and restores from existing state. */ public void restoreFromLastState() { producer.initializeDataModel(dataModel); long latestAnnouncedVersion = announcementWatcher.getLatestVersion(); if (latestAnnouncedVersion == HollowFilesystemAnnouncementWatcher.NO_ANNOUNCEMENT_AVAILABLE || latestAnnouncedVersion < 0) { return; } restore(latestAnnouncedVersion, blobRetriever); }
HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer); incrementalProducer.addOrModify(new TypeA(1, "one", 100)); incrementalProducer.addOrModify(new TypeA(2, "two", 2)); incrementalProducer.addOrModify(new TypeA(3, "three", 300)); incrementalProducer.addOrModify(new TypeA(3, "three", 3)); incrementalProducer.addOrModify(new TypeA(4, "five", 6)); incrementalProducer.delete(new TypeA(5, "five", 5)); incrementalProducer.delete(new TypeB(2, "3")); incrementalProducer.addOrModify(new TypeB(5, "5")); incrementalProducer.addOrModify(new TypeB(5, "6")); incrementalProducer.delete(new RecordPrimaryKey("TypeB", new Object[] { 3 })); Assert.assertTrue(incrementalProducer.hasChanges()); long version = incrementalProducer.runCycle(); Assert.assertTrue(incrementalProducer.hasChanges()); long latestVersion = incrementalProducer.runCycle(); Assert.assertFalse(incrementalProducer.hasChanges()); Assert.assertEquals(1, latestVersion);
@Test public void updateAndPublishUsingMultithreading() { // run within a loop to increase the likelihood of a race condition to occur for (int iterationCounter = 0; iterationCounter < ITERATIONS; ++iterationCounter) { HollowProducer producer = createInMemoryProducer(); /// initialize the data -- classic producer creates the first state in the delta chain. initializeData(producer); /// now we'll be incrementally updating the state by mutating individual records HollowIncrementalProducer incrementalProducer = new HollowIncrementalProducer(producer, THREADS); int[] notModifiedElementIds = IntStream.range(0, ELEMENTS / 2).toArray(); int[] modifiedElementIds = IntStream.range(ELEMENTS / 2, ELEMENTS).toArray(); incrementalProducer.addOrModify(Arrays.stream(modifiedElementIds).mapToObj(i -> new SimpleType(i, i + 1)).collect(Collectors.toList())); /// .runCycle() flushes the changes to a new data state. long versionAfterUpdate = incrementalProducer.runCycle(); /// now we read the changes and assert HollowPrimaryKeyIndex idx = createPrimaryKeyIndex(versionAfterUpdate); Assert.assertFalse(idx.containsDuplicates()); Assert.assertTrue(Arrays.stream(notModifiedElementIds) .boxed() .map(elementId -> getHollowObject(idx, elementId)) .allMatch(obj -> obj.getInt("value") == obj.getInt("id"))); Assert.assertTrue(Arrays.stream(modifiedElementIds) .boxed() .map(elementId -> getHollowObject(idx, elementId)) .allMatch(obj -> obj.getInt("value") != obj.getInt("id"))); } }