How to use

org.apache.samza.util.SystemClock

constructor

 /**
  * create one TaskStorageManager for each task. Add all of them to the
  * List<TaskStorageManager>
  */
 @SuppressWarnings({"unchecked", "rawtypes"})
 private void getContainerStorageManagers() {
  Clock clock = SystemClock.instance();
  StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, clock);
  // don't worry about prefetching for this; looks like the tool doesn't flush to offset files anyways

  Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories();

  for (ContainerModel containerModel : containers.values()) {
   ContainerContext containerContext = new ContainerContextImpl(containerModel, new MetricsRegistryMap());

   ContainerStorageManager containerStorageManager =
     new ContainerStorageManager(containerModel, streamMetadataCache, systemAdmins, changeLogSystemStreams,
       storageEngineFactories, systemFactories, this.getSerdes(), jobConfig, new HashMap<>(),
       new SamzaContainerMetrics(containerModel.getId(), new MetricsRegistryMap()),
       JobContextImpl.fromConfigWithDefaults(jobConfig), containerContext, new HashMap<>(),
       storeBaseDir, storeBaseDir, maxPartitionNumber, new SystemClock());
   this.containerStorageManagers.put(containerModel.getId(), containerStorageManager);
  }
 }
}

systemAdmins,
new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(),
new SystemClock());

@Test
public void joinReverse() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to second stream
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to first stream with same keys
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 int outputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(110, outputSum);
}

@Test
public void joinRetainsMatchedMessagesReverse() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to first stream
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to second stream with same key
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 int outputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(110, outputSum);
 output.clear();
 // push messages to second stream with same keys once again.
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 int newOutputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(110, newOutputSum); // should produce the same output as before
}

systemAdmins,
new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(),
new SystemClock());

systemAdmins,
new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(),
new SystemClock());

systemAdmins,
new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(),
new SystemClock());

@Test
public void joinRetainsLatestMessageForKeyReverse() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to second stream
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to second stream again with same keys but different values
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, 2 * n), messageCollector, taskCoordinator));
 // push messages to first stream with same key
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 int outputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(165, outputSum); // should use latest messages in the second stream
}

@Test
public void join() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to first stream
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to second stream with same keys
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 int outputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(110, outputSum);
}

@Test
public void joinRetainsMatchedMessages() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to first stream
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to second stream with same key
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 int outputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(110, outputSum);
 output.clear();
 // push messages to first stream with same keys once again.
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 int newOutputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(110, newOutputSum); // should produce the same output as before
}

@Test
public void joinNoMatch() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to first stream
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to second stream with different keys
 numbers.forEach(n -> sot.process(new SecondStreamIME(n + 100, n), messageCollector, taskCoordinator));
 assertTrue(output.isEmpty());
}

@Test
public void joinNoMatchReverse() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to second stream
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to first stream with different keys
 numbers.forEach(n -> sot.process(new FirstStreamIME(n + 100, n), messageCollector, taskCoordinator));
 assertTrue(output.isEmpty());
}

@Test
public void joinRetainsLatestMessageForKey() throws Exception {
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 List<Integer> output = new ArrayList<>();
 MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
 // push messages to first stream
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 // push messages to first stream again with same keys but different values
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, 2 * n), messageCollector, taskCoordinator));
 // push messages to second stream with same key
 numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
 int outputSum = output.stream().reduce(0, (s, m) -> s + m);
 assertEquals(165, outputSum); // should use latest messages in the first stream
}

taskInstanceMetrics, samzaContainerMetrics, Mockito.mock(JobContext.class),
Mockito.mock(ContainerContext.class), Mockito.mock(Map.class), DEFAULT_LOGGED_STORE_BASE_DIR,
DEFAULT_STORE_BASE_DIR, 2, new SystemClock());

@Test
public void joinFnInitAndClose() throws Exception {
 TestJoinFunction joinFn = new TestJoinFunction();
 StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(joinFn);
 StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
 MessageCollector messageCollector = mock(MessageCollector.class);
 // push messages to first stream
 numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
 // close should not be called till now
 sot.close();
 verify(messageCollector, times(0)).send(any(OutgoingMessageEnvelope.class));
 // Make sure the joinFn has been copied instead of directly referred by the task instance
 assertEquals(0, joinFn.getNumInitCalls());
 assertEquals(0, joinFn.getNumCloseCalls());
}

@Test(expected = SamzaException.class)
public void joinWithSelfThrowsException() throws Exception {
 Map<String, String> mapConfig = new HashMap<>();
 mapConfig.put("job.name", "jobName");
 mapConfig.put("job.id", "jobId");
 StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream");
 Config config = new MapConfig(mapConfig);
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
   IntegerSerde integerSerde = new IntegerSerde();
   KVSerde<Integer, Integer> kvSerde = KVSerde.of(integerSerde, integerSerde);
   GenericSystemDescriptor sd = new GenericSystemDescriptor("insystem", "mockFactoryClassName");
   GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("inStream", kvSerde);
   MessageStream<KV<Integer, Integer>> inStream = appDesc.getInputStream(inputDescriptor);
   inStream.join(inStream, new TestJoinFunction(), integerSerde, kvSerde, kvSerde, JOIN_TTL, "join");
  }, config);
 createStreamOperatorTask(new SystemClock(), streamAppDesc); // should throw an exception
}