/** * Write the execution plan JSON to a file * @param planJson JSON representation of the plan */ final void writePlanJsonFile(String planJson) { try { String content = "plan='" + planJson + "'"; String planPath = System.getenv(ShellCommandConfig.EXECUTION_PLAN_DIR()); if (planPath != null && !planPath.isEmpty()) { // Write the plan json to plan path File file = new File(planPath + "/plan.json"); file.setReadable(true, false); PrintWriter writer = new PrintWriter(file, "UTF-8"); writer.println(content); writer.close(); } } catch (Exception e) { LOG.warn("Failed to write execution plan json to file", e); } }
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
})); String containerId = System.getenv(ShellCommandConfig.ENV_CONTAINER_ID()); log.info(String.format("Got container ID: %s", containerId)); System.out.println(String.format("Container ID: %s", containerId)); String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); log.info(String.format("Got coordinator URL: %s", coordinatorUrl)); System.out.println(String.format("Coordinator URL: %s", coordinatorUrl));
/** * The entry point for the {@link ClusterBasedJobCoordinator} * @param args args */ public static void main(String[] args) { Config coordinatorSystemConfig = null; final String coordinatorSystemEnv = System.getenv(ShellCommandConfig.ENV_COORDINATOR_SYSTEM_CONFIG()); try { //Read and parse the coordinator system config. log.info("Parsing coordinator system config {}", coordinatorSystemEnv); coordinatorSystemConfig = new MapConfig(SamzaObjectMapper.getObjectMapper().readValue(coordinatorSystemEnv, Config.class)); } catch (IOException e) { log.error("Exception while reading coordinator stream config {}", e); throw new SamzaException(e); } ClusterBasedJobCoordinator jc = new ClusterBasedJobCoordinator(coordinatorSystemConfig); jc.run(); log.info("Finished ClusterBasedJobCoordinator run"); } }
/** * get the config for the AM or containers based on the containers' names. * * @return Config the config of this container */ protected Config getConfig() { Config config; try { if (isApplicationMaster) { config = JobModelManager.currentJobModelManager().jobModel().getConfig(); } else { String url = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String response = HttpUtil.read(new URL(url), 30000, new ExponentialSleepStrategy()); config = SamzaObjectMapper.getObjectMapper().readValue(response, JobModel.class).getConfig(); } } catch (IOException e) { throw new SamzaException("can not read the config", e); } // Make system producer drop producer errors for StreamAppender config = new MapConfig(config, ImmutableMap.of(TaskConfig.DROP_PRODUCER_ERRORS(), "true")); return config; }
/** * * Requests the launch of a StreamProcessor with the specified ID on the resource * @param resource , the SamzaResource on which to launch the StreamProcessor * @param builder, the builder to build the resource launch command from * * TODO: Support non-builder methods to launch resources. Maybe, refactor into a ContainerLaunchStrategy interface */ @Override public void launchStreamProcessor(SamzaResource resource, CommandBuilder builder) { String containerIDStr = builder.buildEnvironment().get(ShellCommandConfig.ENV_CONTAINER_ID()); log.info("Received launch request for {} on hostname {}", containerIDStr, resource.getHost()); synchronized (lock) { try { Container container = allocatedResources.get(resource); if (container == null) { log.info("Resource {} already released. ", resource); return; } runContainer(containerIDStr, container, builder); } catch (Throwable t) { log.error("Error in launching stream processor:", t); clusterManagerCallback.onStreamProcessorLaunchFailure(resource, t); } } }
env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString())); printContainerEnvironmentVariables(samzaContainerId, env);
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
public static void main(String[] args) throws Exception { Thread.setDefaultUncaughtExceptionHandler( new SamzaUncaughtExceptionHandler(() -> { log.info("Exiting process now."); System.exit(1); })); String containerId = System.getenv(ShellCommandConfig.ENV_CONTAINER_ID()); log.info(String.format("Got container ID: %s", containerId)); System.out.println(String.format("Container ID: %s", containerId)); String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); log.info(String.format("Got coordinator URL: %s", coordinatorUrl)); System.out.println(String.format("Coordinator URL: %s", coordinatorUrl)); int delay = new Random().nextInt(SamzaContainer.DEFAULT_READ_JOBMODEL_DELAY_MS()) + 1; JobModel jobModel = SamzaContainer.readJobModel(coordinatorUrl, delay); Config config = jobModel.getConfig(); JobConfig jobConfig = new JobConfig(config); if (jobConfig.getName().isEmpty()) { throw new SamzaException("can not find the job name"); } String jobName = jobConfig.getName().get(); String jobId = jobConfig.getJobId(); MDC.put("containerName", "samza-container-" + containerId); MDC.put("jobName", jobName); MDC.put("jobId", jobId); ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(ApplicationUtil.fromConfig(config), config); run(appDesc, containerId, jobModel, config); System.exit(0); }
/** * The entry point for the {@link ClusterBasedJobCoordinator} * @param args args */ public static void main(String[] args) { Config coordinatorSystemConfig = null; final String coordinatorSystemEnv = System.getenv(ShellCommandConfig.ENV_COORDINATOR_SYSTEM_CONFIG()); try { //Read and parse the coordinator system config. log.info("Parsing coordinator system config {}", coordinatorSystemEnv); coordinatorSystemConfig = new MapConfig(SamzaObjectMapper.getObjectMapper().readValue(coordinatorSystemEnv, Config.class)); } catch (IOException e) { log.error("Exception while reading coordinator stream config {}", e); throw new SamzaException(e); } ClusterBasedJobCoordinator jc = new ClusterBasedJobCoordinator(coordinatorSystemConfig); jc.run(); } }
/** * get the config for the AM or containers based on the containers' names. * * @return Config the config of this container */ protected Config getConfig() { Config config; try { if (isApplicationMaster) { config = JobModelManager.currentJobModelManager().jobModel().getConfig(); } else { String url = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String response = HttpUtil.read(new URL(url), 30000, new ExponentialSleepStrategy()); config = SamzaObjectMapper.getObjectMapper().readValue(response, JobModel.class).getConfig(); } } catch (IOException e) { throw new SamzaException("can not read the config", e); } // Make system producer drop producer errors for StreamAppender config = new MapConfig(config, ImmutableMap.of(TaskConfig.DROP_PRODUCER_ERRORS(), "true")); return config; }
/** * * Requests the launch of a StreamProcessor with the specified ID on the resource * @param resource , the SamzaResource on which to launch the StreamProcessor * @param builder, the builder to build the resource launch command from * * TODO: Support non-builder methods to launch resources. Maybe, refactor into a ContainerLaunchStrategy interface */ @Override public void launchStreamProcessor(SamzaResource resource, CommandBuilder builder) { String containerIDStr = builder.buildEnvironment().get(ShellCommandConfig.ENV_CONTAINER_ID()); log.info("Received launch request for {} on hostname {}", containerIDStr, resource.getHost()); synchronized (lock) { try { Container container = allocatedResources.get(resource); if (container == null) { log.info("Resource {} already released. ", resource); return; } runContainer(containerIDStr, container, builder); } catch (Throwable t) { log.error("Error in launching stream processor:", t); clusterManagerCallback.onStreamProcessorLaunchFailure(resource, t); } } }
env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString())); printContainerEnvironmentVariables(samzaContainerId, env);
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
public static void main(String[] args) throws Exception { Thread.setDefaultUncaughtExceptionHandler( new SamzaUncaughtExceptionHandler(() -> { log.info("Exiting process now."); System.exit(1); })); String containerId = System.getenv(ShellCommandConfig.ENV_CONTAINER_ID()); log.info(String.format("Got container ID: %s", containerId)); System.out.println(String.format("Container ID: %s", containerId)); String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); log.info(String.format("Got coordinator URL: %s", coordinatorUrl)); System.out.println(String.format("Coordinator URL: %s", coordinatorUrl)); int delay = new Random().nextInt(SamzaContainer.DEFAULT_READ_JOBMODEL_DELAY_MS()) + 1; JobModel jobModel = SamzaContainer.readJobModel(coordinatorUrl, delay); Config config = jobModel.getConfig(); JobConfig jobConfig = new JobConfig(config); if (jobConfig.getName().isEmpty()) { throw new SamzaException("can not find the job name"); } String jobName = jobConfig.getName().get(); String jobId = jobConfig.getJobId(); MDC.put("containerName", "samza-container-" + containerId); MDC.put("jobName", jobName); MDC.put("jobId", jobId); ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(ApplicationUtil.fromConfig(config), config); run(appDesc, containerId, jobModel, config); System.exit(0); }
/** * Write the execution plan JSON to a file * @param planJson JSON representation of the plan */ final void writePlanJsonFile(String planJson) { try { String content = "plan='" + planJson + "'"; String planPath = System.getenv(ShellCommandConfig.EXECUTION_PLAN_DIR()); if (planPath != null && !planPath.isEmpty()) { // Write the plan json to plan path File file = new File(planPath + "/plan.json"); file.setReadable(true, false); PrintWriter writer = new PrintWriter(file, "UTF-8"); writer.println(content); writer.close(); } } catch (Exception e) { LOG.warn("Failed to write execution plan json to file", e); } }
/** * The entry point for the {@link ClusterBasedJobCoordinator} * @param args args */ public static void main(String[] args) { Config coordinatorSystemConfig = null; final String coordinatorSystemEnv = System.getenv(ShellCommandConfig.ENV_COORDINATOR_SYSTEM_CONFIG()); try { //Read and parse the coordinator system config. log.info("Parsing coordinator system config {}", coordinatorSystemEnv); coordinatorSystemConfig = new MapConfig(SamzaObjectMapper.getObjectMapper().readValue(coordinatorSystemEnv, Config.class)); } catch (IOException e) { log.error("Exception while reading coordinator stream config {}", e); throw new SamzaException(e); } ClusterBasedJobCoordinator jc = new ClusterBasedJobCoordinator(coordinatorSystemConfig); jc.run(); } }
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
public static void main(String[] args) throws Exception { Thread.setDefaultUncaughtExceptionHandler( new SamzaUncaughtExceptionHandler(() -> { log.info("Exiting process now."); System.exit(1); })); String containerId = System.getenv(ShellCommandConfig.ENV_CONTAINER_ID()); log.info(String.format("Got container ID: %s", containerId)); System.out.println(String.format("Container ID: %s", containerId)); String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); log.info(String.format("Got coordinator URL: %s", coordinatorUrl)); System.out.println(String.format("Coordinator URL: %s", coordinatorUrl)); int delay = new Random().nextInt(SamzaContainer.DEFAULT_READ_JOBMODEL_DELAY_MS()) + 1; JobModel jobModel = SamzaContainer.readJobModel(coordinatorUrl, delay); Config config = jobModel.getConfig(); JobConfig jobConfig = new JobConfig(config); if (jobConfig.getName().isEmpty()) { throw new SamzaException("can not find the job name"); } String jobName = jobConfig.getName().get(); String jobId = jobConfig.getJobId(); MDC.put("containerName", "samza-container-" + containerId); MDC.put("jobName", jobName); MDC.put("jobId", jobId); ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(ApplicationUtil.fromConfig(config), config); run(appDesc, containerId, jobModel, config); System.exit(0); }
/** * Write the execution plan JSON to a file * @param planJson JSON representation of the plan */ final void writePlanJsonFile(String planJson) { try { String content = "plan='" + planJson + "'"; String planPath = System.getenv(ShellCommandConfig.EXECUTION_PLAN_DIR()); if (planPath != null && !planPath.isEmpty()) { // Write the plan json to plan path File file = new File(planPath + "/plan.json"); file.setReadable(true, false); PrintWriter writer = new PrintWriter(file, "UTF-8"); writer.println(content); writer.close(); } } catch (Exception e) { LOG.warn("Failed to write execution plan json to file", e); } }