/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }
/** * Writes the offset files for all side input stores one by one. There is one offset file per store. * Its contents are a JSON encoded mapping from each side input SSP to its last processed offset, and a checksum. */ @VisibleForTesting void writeOffsetFiles() { storeToSSps.entrySet().stream() .filter(entry -> isPersistedStore(entry.getKey())) // filter out in-memory side input stores .forEach((entry) -> { String storeName = entry.getKey(); Map<SystemStreamPartition, String> offsets = entry.getValue().stream() .filter(lastProcessedOffsets::containsKey) .collect(Collectors.toMap(Function.identity(), lastProcessedOffsets::get)); try { String fileContents = OBJECT_WRITER.writeValueAsString(offsets); File offsetFile = new File(getStoreLocation(storeName), OFFSET_FILE); FileUtil.writeWithChecksum(offsetFile, fileContents); } catch (Exception e) { throw new SamzaException("Failed to write offset file for side input store: " + storeName, e); } }); }