@Override public Map<TaskName, Set<SystemStreamPartition>> group(Set<SystemStreamPartition> ssps) { Map<TaskName, Set<SystemStreamPartition>> groupedMap = new HashMap<TaskName, Set<SystemStreamPartition>>(); for (SystemStreamPartition ssp : ssps) { if (broadcastStreams.contains(ssp)) { continue; } HashSet<SystemStreamPartition> sspSet = new HashSet<SystemStreamPartition>(); sspSet.add(ssp); groupedMap.put(new TaskName(ssp.toString()), sspSet); } // assign the broadcast streams to all the taskNames if (!broadcastStreams.isEmpty()) { for (Set<SystemStreamPartition> value : groupedMap.values()) { for (SystemStreamPartition ssp : broadcastStreams) { value.add(ssp); } } } return groupedMap; }
@Override public Map<TaskName, Set<SystemStreamPartition>> group(Set<SystemStreamPartition> ssps) { Map<TaskName, Set<SystemStreamPartition>> groupedMap = new HashMap<TaskName, Set<SystemStreamPartition>>(); for (SystemStreamPartition ssp : ssps) { if (broadcastStreams.contains(ssp)) { continue; } HashSet<SystemStreamPartition> sspSet = new HashSet<SystemStreamPartition>(); sspSet.add(ssp); groupedMap.put(new TaskName(ssp.toString()), sspSet); } // assign the broadcast streams to all the taskNames if (!broadcastStreams.isEmpty()) { for (Set<SystemStreamPartition> value : groupedMap.values()) { value.addAll(broadcastStreams); } } return groupedMap; } }
@Override public Map<TaskName, Set<SystemStreamPartition>> group(Set<SystemStreamPartition> ssps) { Map<TaskName, Set<SystemStreamPartition>> groupedMap = new HashMap<TaskName, Set<SystemStreamPartition>>(); for (SystemStreamPartition ssp : ssps) { if (broadcastStreams.contains(ssp)) { continue; } HashSet<SystemStreamPartition> sspSet = new HashSet<SystemStreamPartition>(); sspSet.add(ssp); groupedMap.put(new TaskName(ssp.toString()), sspSet); } // assign the broadcast streams to all the taskNames if (!broadcastStreams.isEmpty()) { for (Set<SystemStreamPartition> value : groupedMap.values()) { for (SystemStreamPartition ssp : broadcastStreams) { value.add(ssp); } } } return groupedMap; }
@Override public Map<TaskName, Set<SystemStreamPartition>> group(Set<SystemStreamPartition> ssps) { Map<TaskName, Set<SystemStreamPartition>> groupedMap = new HashMap<TaskName, Set<SystemStreamPartition>>(); for (SystemStreamPartition ssp : ssps) { if (broadcastStreams.contains(ssp)) { continue; } HashSet<SystemStreamPartition> sspSet = new HashSet<SystemStreamPartition>(); sspSet.add(ssp); groupedMap.put(new TaskName(ssp.toString()), sspSet); } // assign the broadcast streams to all the taskNames if (!broadcastStreams.isEmpty()) { for (Set<SystemStreamPartition> value : groupedMap.values()) { for (SystemStreamPartition ssp : broadcastStreams) { value.add(ssp); } } } return groupedMap; }
@Override public Map<TaskName, Set<SystemStreamPartition>> group(Set<SystemStreamPartition> ssps) { Map<TaskName, Set<SystemStreamPartition>> groupedMap = new HashMap<TaskName, Set<SystemStreamPartition>>(); for (SystemStreamPartition ssp : ssps) { if (broadcastStreams.contains(ssp)) { continue; } HashSet<SystemStreamPartition> sspSet = new HashSet<SystemStreamPartition>(); sspSet.add(ssp); groupedMap.put(new TaskName(ssp.toString()), sspSet); } // assign the broadcast streams to all the taskNames if (!broadcastStreams.isEmpty()) { for (Set<SystemStreamPartition> value : groupedMap.values()) { for (SystemStreamPartition ssp : broadcastStreams) { value.add(ssp); } } } return groupedMap; }
@Test public void testBroadcastStreamGroupedCorrectly() { Config config = new MapConfig(ImmutableMap.of("task.broadcast.inputs", "SystemA.StreamA#0")); SystemStreamPartitionGrouper grouper = new GroupBySystemStreamPartition(config); Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(ImmutableSet.of(aa0, aa1, aa2, ac0)); Map<TaskName, Set<SystemStreamPartition>> expectedResult = ImmutableMap.<TaskName, Set<SystemStreamPartition>>builder() .put(new TaskName(aa1.toString()), ImmutableSet.of(aa1, aa0)) .put(new TaskName(aa2.toString()), ImmutableSet.of(aa2, aa0)) .put(new TaskName(ac0.toString()), ImmutableSet.of(ac0, aa0)) .build(); assertEquals(expectedResult, result); }
@Test public void testLocalStreamGroupedCorrectly() { SystemStreamPartitionGrouper grouper = grouperFactory.getSystemStreamPartitionGrouper(new MapConfig()); Map<TaskName, Set<SystemStreamPartition>> emptyResult = grouper.group(new HashSet<>()); assertTrue(emptyResult.isEmpty()); Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(ImmutableSet.of(aa0, aa1, aa2, ac0)); Map<TaskName, Set<SystemStreamPartition>> expectedResult = ImmutableMap.<TaskName, Set<SystemStreamPartition>>builder() .put(new TaskName(aa0.toString()), ImmutableSet.of(aa0)) .put(new TaskName(aa1.toString()), ImmutableSet.of(aa1)) .put(new TaskName(aa2.toString()), ImmutableSet.of(aa2)) .put(new TaskName(ac0.toString()), ImmutableSet.of(ac0)) .build(); assertEquals(expectedResult, result); }
@Test public void testGetFileOffsets() { final String storeName = "test-get-file-offsets-store"; final String taskName = "test-get-file-offsets-task"; final String offset = "123"; Set<SystemStreamPartition> ssps = IntStream.range(1, 6) .mapToObj(idx -> new SystemStreamPartition("test-system", "test-stream", new Partition(idx))) .collect(Collectors.toSet()); TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ssps) .build(); initializeSideInputStorageManager(testSideInputStorageManager); ssps.forEach(ssp -> testSideInputStorageManager.updateLastProcessedOffset(ssp, offset)); testSideInputStorageManager.writeOffsetFiles(); Map<SystemStreamPartition, String> fileOffsets = testSideInputStorageManager.getFileOffsets(); ssps.forEach(ssp -> { assertTrue("Failed to get offset for ssp: " + ssp.toString() + " from file.", fileOffsets.containsKey(ssp)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp), offset); }); }
@Test public void testFlush() { final String storeName = "test-flush-store"; final String taskName = "test-flush-task"; final SystemStreamPartition ssp = new SystemStreamPartition("test-system", "test-stream", new Partition(0)); final String offset = "123"; TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ImmutableSet.of(ssp)) .build(); Map<String, StorageEngine> stores = new HashMap<>(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.updateLastProcessedOffset(ssp, offset); testSideInputStorageManager.flush(); for (StorageEngine storageEngine : stores.values()) { verify(storageEngine).flush(); } verify(testSideInputStorageManager).writeOffsetFiles(); File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertTrue("Store directory: " + storeDir.getPath() + " is missing.", storeDir.exists()); Map<SystemStreamPartition, String> fileOffsets = testSideInputStorageManager.getFileOffsets(); assertTrue("Failed to get offset for ssp: " + ssp.toString() + " from file.", fileOffsets.containsKey(ssp)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp), offset); }
@Test public void testWriteOffsetFilesForPersistedStore() { final String storeName = "test-write-offset-persisted-store"; final String storeName2 = "test-write-offset-persisted-store-2"; final String taskName = "test-write-offset-for-persisted-task"; final String offset = "123"; final SystemStreamPartition ssp = new SystemStreamPartition("test-system", "test-stream", new Partition(0)); final SystemStreamPartition ssp2 = new SystemStreamPartition("test-system2", "test-stream2", new Partition(0)); TaskSideInputStorageManager testSideInputStorageManager = new MockTaskSideInputStorageManagerBuilder(taskName, LOGGED_STORE_DIR) .addLoggedStore(storeName, ImmutableSet.of(ssp)) .addLoggedStore(storeName2, ImmutableSet.of(ssp2)) .build(); initializeSideInputStorageManager(testSideInputStorageManager); testSideInputStorageManager.updateLastProcessedOffset(ssp, offset); testSideInputStorageManager.updateLastProcessedOffset(ssp2, offset); testSideInputStorageManager.writeOffsetFiles(); File storeDir = testSideInputStorageManager.getStoreLocation(storeName); assertTrue("Store directory: " + storeDir.getPath() + " is missing.", storeDir.exists()); Map<SystemStreamPartition, String> fileOffsets = testSideInputStorageManager.getFileOffsets(); assertTrue("Failed to get offset for ssp: " + ssp.toString() + " from file.", fileOffsets.containsKey(ssp)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp), offset); assertTrue("Failed to get offset for ssp: " + ssp2.toString() + " from file.", fileOffsets.containsKey(ssp2)); assertEquals("Mismatch between last processed offset and file offset.", fileOffsets.get(ssp2), offset); }