/** * create one TaskStorageManager for each task. Add all of them to the * List<TaskStorageManager> */ @SuppressWarnings({"unchecked", "rawtypes"}) private void getContainerStorageManagers() { Clock clock = SystemClock.instance(); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, clock); // don't worry about prefetching for this; looks like the tool doesn't flush to offset files anyways Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (ContainerModel containerModel : containers.values()) { ContainerContext containerContext = new ContainerContextImpl(containerModel, new MetricsRegistryMap()); ContainerStorageManager containerStorageManager = new ContainerStorageManager(containerModel, streamMetadataCache, systemAdmins, changeLogSystemStreams, storageEngineFactories, systemFactories, this.getSerdes(), jobConfig, new HashMap<>(), new SamzaContainerMetrics(containerModel.getId(), new MetricsRegistryMap()), JobContextImpl.fromConfigWithDefaults(jobConfig), containerContext, new HashMap<>(), storeBaseDir, storeBaseDir, maxPartitionNumber, new SystemClock()); this.containerStorageManagers.put(containerModel.getId(), containerStorageManager); } } }
systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
@Test public void joinReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to second stream numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to first stream with same keys numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, outputSum); }
@Test public void joinRetainsMatchedMessagesReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream with same key numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, outputSum); output.clear(); // push messages to second stream with same keys once again. numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int newOutputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, newOutputSum); // should produce the same output as before }
systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
@Test public void joinRetainsLatestMessageForKeyReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to second stream numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream again with same keys but different values numbers.forEach(n -> sot.process(new SecondStreamIME(n, 2 * n), messageCollector, taskCoordinator)); // push messages to first stream with same key numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(165, outputSum); // should use latest messages in the second stream }
@Test public void join() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream with same keys numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, outputSum); }
@Test public void joinRetainsMatchedMessages() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream with same key numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, outputSum); output.clear(); // push messages to first stream with same keys once again. numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); int newOutputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, newOutputSum); // should produce the same output as before }
@Test public void joinNoMatch() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream with different keys numbers.forEach(n -> sot.process(new SecondStreamIME(n + 100, n), messageCollector, taskCoordinator)); assertTrue(output.isEmpty()); }
@Test public void joinNoMatchReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to second stream numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to first stream with different keys numbers.forEach(n -> sot.process(new FirstStreamIME(n + 100, n), messageCollector, taskCoordinator)); assertTrue(output.isEmpty()); }
@Test public void joinRetainsLatestMessageForKey() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to first stream again with same keys but different values numbers.forEach(n -> sot.process(new FirstStreamIME(n, 2 * n), messageCollector, taskCoordinator)); // push messages to second stream with same key numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(165, outputSum); // should use latest messages in the first stream }
taskInstanceMetrics, samzaContainerMetrics, Mockito.mock(JobContext.class), Mockito.mock(ContainerContext.class), Mockito.mock(Map.class), DEFAULT_LOGGED_STORE_BASE_DIR, DEFAULT_STORE_BASE_DIR, 2, new SystemClock());
@Test public void joinFnInitAndClose() throws Exception { TestJoinFunction joinFn = new TestJoinFunction(); StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(joinFn); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); MessageCollector messageCollector = mock(MessageCollector.class); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // close should not be called till now sot.close(); verify(messageCollector, times(0)).send(any(OutgoingMessageEnvelope.class)); // Make sure the joinFn has been copied instead of directly referred by the task instance assertEquals(0, joinFn.getNumInitCalls()); assertEquals(0, joinFn.getNumCloseCalls()); }
@Test(expected = SamzaException.class) public void joinWithSelfThrowsException() throws Exception { Map<String, String> mapConfig = new HashMap<>(); mapConfig.put("job.name", "jobName"); mapConfig.put("job.id", "jobId"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream"); Config config = new MapConfig(mapConfig); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { IntegerSerde integerSerde = new IntegerSerde(); KVSerde<Integer, Integer> kvSerde = KVSerde.of(integerSerde, integerSerde); GenericSystemDescriptor sd = new GenericSystemDescriptor("insystem", "mockFactoryClassName"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("inStream", kvSerde); MessageStream<KV<Integer, Integer>> inStream = appDesc.getInputStream(inputDescriptor); inStream.join(inStream, new TestJoinFunction(), integerSerde, kvSerde, kvSerde, JOIN_TTL, "join"); }, config); createStreamOperatorTask(new SystemClock(), streamAppDesc); // should throw an exception }