How to use

shutdown

method

in

org.apache.samza.container.SamzaContainer

/**
 * Stops the {@link SamzaContainer}.
 * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise.
 */
private boolean stopSamzaContainer() {
 boolean hasContainerShutdown = true;
 if (container != null) {
  try {
   container.shutdown();
   LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container);
   hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS);
  } catch (InterruptedException e) {
   LOGGER.error("Exception occurred when shutting down the container: {}.", container, e);
   hasContainerShutdown = false;
   if (containerException != null) {
    containerException = e;
   }
  }
  LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown));
 }
 // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception
 // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution.
 // It is acceptable since container exception is much more useful compared to timeout exception.
 // We can infer from the logs about the fact that container shutdown timed out or not for additional inference.
 if (!hasContainerShutdown && containerException == null) {
  containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms.");
 }
 return hasContainerShutdown;
}

/**
 * Stops the {@link SamzaContainer}.
 * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise.
 */
private boolean stopSamzaContainer() {
 boolean hasContainerShutdown = true;
 if (container != null) {
  try {
   container.shutdown();
   LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container);
   hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS);
  } catch (InterruptedException e) {
   LOGGER.error("Exception occurred when shutting down the container: {}.", container, e);
   hasContainerShutdown = false;
   if (containerException != null) {
    containerException = e;
   }
  }
  LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown));
 }
 // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception
 // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution.
 // It is acceptable since container exception is much more useful compared to timeout exception.
 // We can infer from the logs about the fact that container shutdown timed out or not for additional inference.
 if (!hasContainerShutdown && containerException == null) {
  containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms.");
 }
 return hasContainerShutdown;
}

/**
 * Stops the {@link SamzaContainer}.
 * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise.
 */
private boolean stopSamzaContainer() {
 boolean hasContainerShutdown = true;
 if (container != null) {
  try {
   container.shutdown();
   LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container);
   hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS);
  } catch (InterruptedException e) {
   LOGGER.error("Exception occurred when shutting down the container: {}.", container, e);
   hasContainerShutdown = false;
   if (containerException != null) {
    containerException = e;
   }
  }
  LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown));
 }
 // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception
 // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution.
 // It is acceptable since container exception is much more useful compared to timeout exception.
 // We can infer from the logs about the fact that container shutdown timed out or not for additional inference.
 if (!hasContainerShutdown && containerException == null) {
  containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms.");
 }
 return hasContainerShutdown;
}

/**
 * Stops the {@link SamzaContainer}.
 * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise.
 */
private boolean stopSamzaContainer() {
 boolean hasContainerShutdown = true;
 if (container != null) {
  try {
   container.shutdown();
   LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container);
   hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS);
  } catch (InterruptedException e) {
   LOGGER.error("Exception occurred when shutting down the container: {}.", container, e);
   hasContainerShutdown = false;
   if (containerException != null) {
    containerException = e;
   }
  }
  LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown));
 }
 // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception
 // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution.
 // It is acceptable since container exception is much more useful compared to timeout exception.
 // We can infer from the logs about the fact that container shutdown timed out or not for additional inference.
 if (!hasContainerShutdown && containerException == null) {
  containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms.");
 }
 return hasContainerShutdown;
}

/**
 * Stops the {@link SamzaContainer}.
 * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise.
 */
private boolean stopSamzaContainer() {
 boolean hasContainerShutdown = true;
 if (container != null) {
  try {
   container.shutdown();
   LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container);
   hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS);
  } catch (InterruptedException e) {
   LOGGER.error("Exception occurred when shutting down the container: {}.", container, e);
   hasContainerShutdown = false;
   if (containerException != null) {
    containerException = e;
   }
  }
  LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown));
 }
 // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception
 // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution.
 // It is acceptable since container exception is much more useful compared to timeout exception.
 // We can infer from the logs about the fact that container shutdown timed out or not for additional inference.
 if (!hasContainerShutdown && containerException == null) {
  containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms.");
 }
 return hasContainerShutdown;
}

 /**
  * Creates a new container heartbeat monitor if possible.
  * @param container the container to monitor
  * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one
  */
 private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) {
  String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
  String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID());
  if (executionEnvContainerId != null) {
   log.info("Got execution environment container id: {}", executionEnvContainerId);
   return new ContainerHeartbeatMonitor(() -> {
     try {
      container.shutdown();
      containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat");
     } catch (Exception e) {
      log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e);
      System.exit(1);
     }
    }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId));
  } else {
   log.warn("Execution environment container id not set. Container heartbeat monitor will not be created");
   return null;
  }
 }
}

 /**
  * Creates a new container heartbeat monitor if possible.
  * @param container the container to monitor
  * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one
  */
 private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) {
  String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
  String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID());
  if (executionEnvContainerId != null) {
   log.info("Got execution environment container id: {}", executionEnvContainerId);
   return new ContainerHeartbeatMonitor(() -> {
     try {
      container.shutdown();
      containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat");
     } catch (Exception e) {
      log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e);
      System.exit(1);
     }
    }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId));
  } else {
   log.warn("Execution environment container id not set. Container heartbeat monitor will not be created");
   return null;
  }
 }
}

 /**
  * Creates a new container heartbeat monitor if possible.
  * @param container the container to monitor
  * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one
  */
 private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) {
  String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
  String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID());
  if (executionEnvContainerId != null) {
   log.info("Got execution environment container id: {}", executionEnvContainerId);
   return new ContainerHeartbeatMonitor(() -> {
     try {
      container.shutdown();
      containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat");
     } catch (Exception e) {
      log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e);
      System.exit(1);
     }
    }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId));
  } else {
   log.warn("Execution environment container id not set. Container heartbeat monitor will not be created");
   return null;
  }
 }
}

 /**
  * Creates a new container heartbeat monitor if possible.
  * @param container the container to monitor
  * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one
  */
 private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) {
  String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
  String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID());
  if (executionEnvContainerId != null) {
   log.info("Got execution environment container id: {}", executionEnvContainerId);
   return new ContainerHeartbeatMonitor(() -> {
     try {
      container.shutdown();
      containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat");
     } catch (Exception e) {
      log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e);
      System.exit(1);
     }
    }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId));
  } else {
   log.warn("Execution environment container id not set. Container heartbeat monitor will not be created");
   return null;
  }
 }
}

 /**
  * Creates a new container heartbeat monitor if possible.
  * @param container the container to monitor
  * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one
  */
 private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) {
  String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
  String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID());
  if (executionEnvContainerId != null) {
   log.info("Got execution environment container id: {}", executionEnvContainerId);
   return new ContainerHeartbeatMonitor(() -> {
     try {
      container.shutdown();
      containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat");
     } catch (Exception e) {
      log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e);
      System.exit(1);
     }
    }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId));
  } else {
   log.warn("Execution environment container id not set. Container heartbeat monitor will not be created");
   return null;
  }
 }
}

@Test
public void testCoordinatorFailureShouldStopTheStreamProcessor() {
 JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
 ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
 SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
 MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
 StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, lifecycleListener, mockJobCoordinator);
 Exception failureException = new Exception("dummy exception");
 streamProcessor.container = mockSamzaContainer;
 streamProcessor.state = State.RUNNING;
 streamProcessor.jobCoordinatorListener.onCoordinatorFailure(failureException);
 Mockito.doNothing().when(mockSamzaContainer).shutdown();
 Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
 assertEquals(State.STOPPED, streamProcessor.state);
 Mockito.verify(lifecycleListener).afterFailure(failureException);
 Mockito.verify(mockSamzaContainer).shutdown();
}

@Test
public void testStopShouldBeIdempotent() {
 JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class);
 ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class);
 SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class);
 MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0"));
 StreamProcessor streamProcessor = PowerMockito.spy(new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, lifecycleListener, mockJobCoordinator));
 Mockito.doNothing().when(mockJobCoordinator).stop();
 Mockito.doNothing().when(mockSamzaContainer).shutdown();
 Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
 Mockito.when(mockSamzaContainer.getStatus())
     .thenReturn(SamzaContainerStatus.STARTED)
     .thenReturn(SamzaContainerStatus.STOPPED);
 streamProcessor.state = State.RUNNING;
 streamProcessor.stop();
 assertEquals(State.STOPPING, streamProcessor.state);
}

Mockito.doNothing().when(mockSamzaContainer).shutdown();
Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false);
Mockito.when(mockSamzaContainer.getStatus())
Mockito.verify(mockSamzaContainer, Mockito.times(1)).shutdown();
Mockito.verify(mockJobCoordinator, Mockito.times(1)).stop();