/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Initializes the store directories for all the stores: * 1. Cleans up the directories for invalid stores. * 2. Ensures that the directories exist. */ private void initializeStoreDirectories() { LOG.info("Initializing side input store directories."); stores.keySet().forEach(storeName -> { File storeLocation = getStoreLocation(storeName); String storePath = storeLocation.toPath().toString(); if (!isValidSideInputStore(storeName, storeLocation)) { LOG.info("Cleaning up the store directory at {} for {}", storePath, storeName); FileUtil.rm(storeLocation); } if (isPersistedStore(storeName) && !storeLocation.exists()) { LOG.info("Creating {} as the store directory for the side input store {}", storePath, storeName); storeLocation.mkdirs(); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }
private boolean isValidSideInputStore(String storeName, File storeLocation) { return isPersistedStore(storeName) && !StorageManagerUtil.isStaleStore(storeLocation, OFFSET_FILE, STORE_DELETE_RETENTION_MS, clock.currentTimeMillis()) && StorageManagerUtil.isOffsetFileValid(storeLocation, OFFSET_FILE); }