/** * Initializes the side input storage manager. */ public void init() { LOG.info("Initializing side input stores."); Map<SystemStreamPartition, String> fileOffsets = getFileOffsets(); LOG.info("File offsets for the task {}: ", taskName, fileOffsets); Map<SystemStreamPartition, String> oldestOffsets = getOldestOffsets(); LOG.info("Oldest offsets for the task {}: ", taskName, fileOffsets); startingOffsets = getStartingOffsets(fileOffsets, oldestOffsets); LOG.info("Starting offsets for the task {}: {}", taskName, startingOffsets); lastProcessedOffsets.putAll(fileOffsets); LOG.info("Last processed offsets for the task {}: {}", taskName, lastProcessedOffsets); initializeStoreDirectories(); }
/** * Initializes the side input storage manager. */ public void init() { LOG.info("Initializing side input stores."); Map<SystemStreamPartition, String> fileOffsets = getFileOffsets(); LOG.info("File offsets for the task {}: ", taskName, fileOffsets); Map<SystemStreamPartition, String> oldestOffsets = getOldestOffsets(); LOG.info("Oldest offsets for the task {}: ", taskName, fileOffsets); startingOffsets = getStartingOffsets(fileOffsets, oldestOffsets); LOG.info("Starting offsets for the task {}: {}", taskName, startingOffsets); lastProcessedOffsets.putAll(fileOffsets); LOG.info("Last processed offsets for the task {}: {}", taskName, lastProcessedOffsets); initializeStoreDirectories(); }
/** * Initializes the side input storage manager. */ public void init() { LOG.info("Initializing side input stores."); Map<SystemStreamPartition, String> fileOffsets = getFileOffsets(); LOG.info("File offsets for the task {}: ", taskName, fileOffsets); Map<SystemStreamPartition, String> oldestOffsets = getOldestOffsets(); LOG.info("Oldest offsets for the task {}: ", taskName, fileOffsets); startingOffsets = getStartingOffsets(fileOffsets, oldestOffsets); LOG.info("Starting offsets for the task {}: {}", taskName, startingOffsets); lastProcessedOffsets.putAll(fileOffsets); LOG.info("Last processed offsets for the task {}: {}", taskName, lastProcessedOffsets); initializeStoreDirectories(); }
/** * Initializes the side input storage manager. */ public void init() { LOG.info("Initializing side input stores."); Map<SystemStreamPartition, String> fileOffsets = getFileOffsets(); LOG.info("File offsets for the task {}: ", taskName, fileOffsets); Map<SystemStreamPartition, String> oldestOffsets = getOldestOffsets(); LOG.info("Oldest offsets for the task {}: ", taskName, fileOffsets); startingOffsets = getStartingOffsets(fileOffsets, oldestOffsets); LOG.info("Starting offsets for the task {}: {}", taskName, startingOffsets); lastProcessedOffsets.putAll(fileOffsets); LOG.info("Last processed offsets for the task {}: {}", taskName, lastProcessedOffsets); initializeStoreDirectories(); }
/** * Initializes the side input storage manager. */ public void init() { LOG.info("Initializing side input stores."); Map<SystemStreamPartition, String> fileOffsets = getFileOffsets(); LOG.info("File offsets for the task {}: ", taskName, fileOffsets); Map<SystemStreamPartition, String> oldestOffsets = getOldestOffsets(); LOG.info("Oldest offsets for the task {}: ", taskName, fileOffsets); startingOffsets = getStartingOffsets(fileOffsets, oldestOffsets); LOG.info("Starting offsets for the task {}: {}", taskName, startingOffsets); lastProcessedOffsets.putAll(fileOffsets); LOG.info("Last processed offsets for the task {}: {}", taskName, lastProcessedOffsets); initializeStoreDirectories(); }
private void initializeSideInputStorageManager(TaskSideInputStorageManager testSideInputStorageManager) { doReturn(new HashMap<>()).when(testSideInputStorageManager).getStartingOffsets(any(), any()); testSideInputStorageManager.init(); }
@Test public void testGetStartingOffsets() { final String storeName = "test-get-starting-offset-store"; final String taskName = "test-get-starting-offset-task"; Set<SystemStreamPartition> ssps = IntStream.range(1, 6) .mapToObj(idx -> new SystemStreamPartition("test-system", "test-stream", new Partition(idx))) .collect(Collectors.toSet()); TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ssps) .build(); initializeSideInputStorageManager(testSideInputStorageManager); Map<SystemStreamPartition, String> fileOffsets = ssps.stream() .collect(Collectors.toMap(Function.identity(), ssp -> { int partitionId = ssp.getPartition().getPartitionId(); int offset = partitionId % 2 == 0 ? partitionId + 10 : partitionId; return String.valueOf(offset); })); Map<SystemStreamPartition, String> oldestOffsets = ssps.stream() .collect(Collectors.toMap(Function.identity(), ssp -> { int partitionId = ssp.getPartition().getPartitionId(); int offset = partitionId % 2 == 0 ? partitionId : partitionId + 10; return String.valueOf(offset); })); doCallRealMethod().when(testSideInputStorageManager).getStartingOffsets(fileOffsets, oldestOffsets); Map<SystemStreamPartition, String> startingOffsets = testSideInputStorageManager.getStartingOffsets(fileOffsets, oldestOffsets); assertTrue("Failed to get starting offsets for all ssps", startingOffsets.size() == 5); }