public TaskRestoreManager(TaskModel taskModel, Map<String, SystemStream> changelogSystemStreams, Map<String, StorageEngine> taskStores, SystemAdmins systemAdmins, Clock clock) { this.taskStores = taskStores; this.taskModel = taskModel; this.clock = clock; this.changelogSystemStreams = changelogSystemStreams; this.systemAdmins = systemAdmins; this.fileOffsets = new HashMap<>(); this.taskStoresToRestore = this.taskStores.entrySet().stream() .filter(x -> x.getValue().getStoreProperties().isLoggedStore()) .map(x -> x.getKey()).collect(Collectors.toSet()); }
/** * Stop only persistent stores. In case of certain stores and store mode (such as RocksDB), this * can invoke compaction. */ public void stopPersistentStores() { this.taskStores.values().stream().filter(storageEngine -> { return storageEngine.getStoreProperties().isPersistedToDisk(); }).forEach(storageEngine -> { storageEngine.stop(); }); } }
storeRestoreCallCount++; return null; }).when(mockStorageEngine).restore(Mockito.any()); }).when(mockStorageEngine).getStoreProperties();
/** * Stop all stores. */ public void stop() { this.taskStores.values().forEach(storageEngine -> { storageEngine.stop(); }); }
/** * Restore each store in taskStoresToRestore sequentially */ public void restoreStores() { LOG.debug("Restoring stores for task: {}", taskModel.getTaskName()); for (String storeName : taskStoresToRestore) { SystemConsumer systemConsumer = systemConsumers.get(storeName); SystemStream systemStream = changelogSystemStreams.get(storeName); SystemStreamPartitionIterator systemStreamPartitionIterator = new SystemStreamPartitionIterator(systemConsumer, new SystemStreamPartition(systemStream, taskModel.getChangelogPartition())); taskStores.get(storeName).restore(systemStreamPartitionIterator); } }
@Test public void testFlush() { final String storeName = "test-flush-store"; final String taskName = "test-flush-task"; final SystemStreamPartition ssp = new SystemStreamPartition("test-system", "test-stream", new Partition(0)); final String offset = "123"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ImmutableSet.of(ssp)) .build(); Map<String, StorageEngine> stores = new HashMap<>(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.updateLastProcessedOffset(ssp, offset); testSideInputStorageManager.flush(); for (StorageEngine storageEngine : stores.values()) { verify(storageEngine).flush(); } verify(testSideInputStorageManager).writeOffsetFiles(); File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertTrue("Store directory: " + storeDir.getPath() + " is missing.", storeDir.exists()); Map<SystemStreamPartition, String> fileOffsets = testSideInputStorageManager.getFileOffsets(); assertTrue("Failed to get offset for ssp: " + ssp.toString() + " from file.", fileOffsets.containsKey(ssp)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp), offset); }
@VisibleForTesting public void stopStores() { this.taskStores.forEach((taskName, storeMap) -> storeMap.forEach((storeName, store) -> store.stop())); }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
@Test public void testStop() { final String storeName = "test-stop-store"; final String taskName = "test-stop-task"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, NON_LOGGED_STORE_DIR) .addInMemoryStore(storeName, ImmutableSet.of()) .build(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.stop(); verify(testSideInputStorageManager.getStore(storeName)).stop(); verify(testSideInputStorageManager).writeOffsetFiles(); }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
private void validateStoreConfiguration() { stores.forEach((storeName, storageEngine) -> { if (!storeToProcessor.containsKey(storeName)) { throw new SamzaException( String.format("Side inputs processor missing for store: %s.", storeName)); } if (storageEngine.getStoreProperties().isLoggedStore()) { throw new SamzaException( String.format("Cannot configure both side inputs and a changelog for store: %s.", storeName)); } }); } }
.getStoreProperties() .isPersistedToDisk()) {
/** * Create stores' base directories for logged-stores if they dont exist. */ private void setupBaseDirs() { LOG.debug("Setting up base directories for stores."); taskStores.forEach((storeName, storageEngine) -> { if (storageEngine.getStoreProperties().isLoggedStore()) { File loggedStorePartitionDir = StorageManagerUtil.getStorePartitionDir(loggedStoreBaseDirectory, storeName, taskModel.getTaskName()); LOG.info("Using logged storage partition directory: " + loggedStorePartitionDir.toPath().toString() + " for store: " + storeName); if (!loggedStorePartitionDir.exists()) { loggedStorePartitionDir.mkdirs(); } } else { File nonLoggedStorePartitionDir = StorageManagerUtil.getStorePartitionDir(nonLoggedStoreBaseDirectory, storeName, taskModel.getTaskName()); LOG.info("Using non logged storage partition directory: " + nonLoggedStorePartitionDir.toPath().toString() + " for store: " + storeName); nonLoggedStorePartitionDir.mkdirs(); } }); }
MockTaskSideInputStorageManagerBuilder addInMemoryStore(String storeName, Set<SystemStreamPartition> ssps) { StorageEngine storageEngine = mock(StorageEngine.class); when(storageEngine.getStoreProperties()).thenReturn( new StoreProperties.StorePropertiesBuilder().setLoggedStore(false).setPersistedToDisk(false).build()); stores.put(storeName, storageEngine); storeToProcessor.put(storeName, mock(SideInputsProcessor.class)); storeToSSps.put(storeName, ssps); return this; }
MockTaskSideInputStorageManagerBuilder addLoggedStore(String storeName, Set<SystemStreamPartition> ssps) { StorageEngine storageEngine = mock(StorageEngine.class); when(storageEngine.getStoreProperties()).thenReturn( new StoreProperties.StorePropertiesBuilder().setLoggedStore(false).setPersistedToDisk(true).build()); stores.put(storeName, storageEngine); storeToProcessor.put(storeName, mock(SideInputsProcessor.class)); storeToSSps.put(storeName, ssps); return this; }
/** * Directory loggedStoreDir associated with the logged store storeName is determined to be valid * if all of the following conditions are true. * a) If the store has to be persisted to disk. * b) If there is a valid offset file associated with the logged store. * c) If the logged store has not gone stale. * * @return true if the logged store is valid, false otherwise. */ private boolean isLoggedStoreValid(String storeName, File loggedStoreDir) { long changeLogDeleteRetentionInMs = StorageConfig.DEFAULT_CHANGELOG_DELETE_RETENTION_MS(); if (new StorageConfig(config).getChangeLogDeleteRetentionsInMs().get(storeName).isDefined()) { changeLogDeleteRetentionInMs = (long) new StorageConfig(config).getChangeLogDeleteRetentionsInMs().get(storeName).get(); } return this.taskStores.get(storeName).getStoreProperties().isPersistedToDisk() && StorageManagerUtil.isOffsetFileValid(loggedStoreDir, OFFSET_FILE_NAME) && !StorageManagerUtil.isStaleStore( loggedStoreDir, OFFSET_FILE_NAME, changeLogDeleteRetentionInMs, clock.currentTimeMillis()); }