@Override public HollowProducer.Blob openSnapshot(long version) { return new FilesystemBlob(HollowConstants.VERSION_NONE, version, SNAPSHOT, stagingPath, compressor); }
@Override public Blob openReverseDelta(long fromVersion, long toVersion) { return new InMemoryBlob(fromVersion, toVersion, Blob.Type.REVERSE_DELTA); }
/** * Cleans snapshot to keep the last 'n' snapshots. Defaults to 5. */ @Override public void cleanSnapshots() { File[] files = getFilesByType(HollowProducer.Blob.Type.SNAPSHOT.prefix); if(files == null || files.length <= numOfSnapshotsToKeep) { return; } sortByLastModified(files); for(int i= numOfSnapshotsToKeep; i < files.length; i++){ File file = files[i]; boolean deleted = file.delete(); if(!deleted) { log.warning("Could not delete snapshot " + file.getPath()); } } }
private HollowProducer createInMemoryProducer() { return HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); }
protected void checkArguments() { if (stager != null && compressor != null) { throw new IllegalArgumentException( "Both a custom BlobStager and BlobCompressor were specified -- please specify only one of these."); } if (stager != null && stagingDir != null) { throw new IllegalArgumentException( "Both a custom BlobStager and a staging directory were specified -- please specify only one of these."); } if (this.stager == null) { BlobCompressor compressor = this.compressor != null ? this.compressor : BlobCompressor.NO_COMPRESSION; File stagingDir = this.stagingDir != null ? this.stagingDir : new File( System.getProperty("java.io.tmpdir")); this.stager = new HollowFilesystemBlobStager(stagingDir.toPath(), compressor); } }
private HollowProducer createProducer(File tmpFolder, HollowObjectSchema... schemas) { HollowProducer producer = HollowProducer.withPublisher(new FakeBlobPublisher()) .withAnnouncer(new HollowFilesystemAnnouncer(tmpFolder.toPath())).build(); if (schemas != null && schemas.length > 0) { producer.initializeDataModel(schemas); } producer.addListener(new FakeProducerListener()); return producer; }
private HollowProducer createInMemoryProducer() { return HollowProducer.withPublisher(blobStore).withBlobStager(new HollowInMemoryBlobStager()).build(); }
@Override public HollowProducer.Blob openDelta(long fromVersion, long toVersion){ return new FilesystemBlob(fromVersion, toVersion, DELTA, stagingPath, compressor); }
@Override public Blob openSnapshot(long version) { return new InMemoryBlob(HollowConstants.VERSION_NONE, version, Blob.Type.SNAPSHOT); }
@Deprecated public HollowProducer( Publisher publisher, Announcer announcer) { this(new HollowFilesystemBlobStager(), publisher, announcer, Collections.emptyList(), new VersionMinterWithCounter(), null, 0, DEFAULT_TARGET_MAX_TYPE_SHARD_SIZE, null, new DummyBlobStorageCleaner(), new BasicSingleProducerEnforcer()); }
private static HollowProducer getProducer(ObjectModificationValidator validator) { return HollowProducer.withPublisher(new InMemoryBlobStore()) .withBlobStager(new HollowInMemoryBlobStager()) .withListener(validator) .build(); }
@Override public HollowProducer.Blob openReverseDelta(long fromVersion, long toVersion) { return new FilesystemBlob(fromVersion, toVersion, REVERSE_DELTA, stagingPath, compressor); }
@Override public Blob openDelta(long fromVersion, long toVersion) { return new InMemoryBlob(fromVersion, toVersion, Blob.Type.DELTA); }
@Test(expected = NullPointerException.class) public void failsWhenNotEnoughArgs() { HollowProducer backingProducer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); //No AnnouncementWatcher, BlobRetriever and DataModel to restore HollowIncrementalProducer incrementalProducer = HollowIncrementalProducer.withProducer(backingProducer) .build(); incrementalProducer.restoreFromLastState(); }
@Test public void testNullMetricsCollector() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); long version = producer.runCycle(new HollowProducer.Populator() { public void populate(HollowProducer.WriteState state) throws Exception { state.add(Integer.valueOf(1)); } }); HollowConsumer consumer = HollowConsumer.withBlobRetriever(blobStore) .withMetricsCollector(null) .build(); consumer.triggerRefreshTo(version); }
@Test(expected = RuntimeException.class) public void failsWhenLastStateIsNotAvailable() { HollowProducer backingProducer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); HollowIncrementalProducer incrementalProducer = HollowIncrementalProducer.withProducer(backingProducer) .withBlobRetriever(blobStore) .withAnnouncementWatcher(new FakeAnnouncementWatcher(0)) .withDataModel(TypeA.class, TypeB.class) .build(); incrementalProducer.restoreFromLastState(); }
@Test public void producerUsesCustomVersionMinter() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .withVersionMinter(new VersionMinter() { long counter = 0; public long mint() { return ++counter; } }) .build(); long v1 = runCycle(producer, 1); long v2 = runCycle(producer, 2); long v3 = runCycle(producer, 3); Assert.assertEquals(1, v1); Assert.assertEquals(2, v2); Assert.assertEquals(3, v3); }
@Test public void publishAndLoadASnapshot() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); /// Showing verbose version of `runCycle(producer, 1);` long version = producer.runCycle(state -> state.add(1)); HollowConsumer consumer = HollowConsumer.withBlobRetriever(blobStore).build(); consumer.triggerRefreshTo(version); Assert.assertEquals(version, consumer.getCurrentVersionId()); }
@Test public void metricsWhenPublishingFails() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); try { producer.runCycle(new HollowProducer.Populator() { public void populate(HollowProducer.WriteState state) throws Exception { state.add(null); } }); } catch (Exception ignored){ } HollowProducerMetrics hollowProducerMetrics = producer.getMetrics(); Assert.assertEquals(hollowProducerMetrics.getCyclesSucceeded(), 0); Assert.assertEquals(hollowProducerMetrics.getCyclesCompleted(), 1); Assert.assertEquals(hollowProducerMetrics.getCycleFailed(), 1); } }
@Test public void metricsWhenPublishingSnapshot() { HollowProducer producer = HollowProducer.withPublisher(blobStore) .withBlobStager(new HollowInMemoryBlobStager()) .build(); producer.runCycle(new HollowProducer.Populator() { public void populate(HollowProducer.WriteState state) throws Exception { state.add(Integer.valueOf(1)); } }); HollowProducerMetrics hollowProducerMetrics = producer.getMetrics(); Assert.assertEquals(hollowProducerMetrics.getCyclesSucceeded(), 1); Assert.assertEquals(hollowProducerMetrics.getCyclesCompleted(), 1); Assert.assertEquals(hollowProducerMetrics.getTotalPopulatedOrdinals(), 1); Assert.assertEquals(hollowProducerMetrics.getSnapshotsCompleted(), 1); }