/** * Given a Twill app name, returns the id of the program that was used to construct this Twill app name. * * @throws IllegalArgumentException if the given app name does not match the {@link #APP_NAME_PATTERN}. */ public static ProgramId fromTwillAppName(String twillAppName) { return fromTwillAppName(twillAppName, true); }
@Override public void completed() { super.completed(); // On normal AM completion, based on the last container failure to publish the state if (lastContainerFailure == null) { programStateWriterWithHeartBeat.completed(); } else { lastContainerFailure.writeError(programStateWriterWithHeartBeat); } }
@Override protected void configure() { bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class); } }
@Override public boolean accept(String className, URL classUrl, URL classPathUrl) { // Exclude both hadoop and spark classes. if (urls.contains(classPathUrl)) { return false; } return super.accept(className, classUrl, classPathUrl) && !className.startsWith("org.apache.spark."); } };
@Override public void containerStopped(String runnableName, int instanceId, String containerId, int exitStatus) { super.containerStopped(runnableName, instanceId, containerId, exitStatus); // Let the completed() method handle when a container has completed with no error if (exitStatus == 0) { return; } switch(programRunId.getType()) { case WORKFLOW: case SPARK: case MAPREDUCE: // For workflow, MapReduce, and spark, if there is an error, the program state is failure // We defer the actual publish to one of the completion methods (killed, completed, aborted) // as we need to know under what condition the container failed. lastContainerFailure = new ContainerFailure(runnableName, instanceId, containerId, exitStatus); break; default: // For other programs, the container will be re-launched - the program state will continue to be RUNNING // TODO Workers should be configured via runtime args // to support both retrying on failure, or just failing and not retrying. break; } }
@Override public void killed() { super.killed(); // The AM is stopped explicitly, always record the state as killed. programStateWriterWithHeartBeat.killed(); }
/** * Creates the {@link EventHandler} for handling the application events. */ @Nullable private EventHandler createEventHandler(CConfiguration cConf, ProgramRunId programRunId) { if (clusterMode != ClusterMode.ON_PREMISE) { return null; } return new TwillAppLifecycleEventHandler(cConf.getLong(Constants.CFG_TWILL_NO_CONTAINER_TIMEOUT, Long.MAX_VALUE), this instanceof LongRunningDistributedProgramRunner, programRunId); }
private int getSystemServiceInstanceCount(String serviceName) throws Exception { Integer count = serviceStore.getServiceInstance(serviceName); //In standalone mode, this count will be null. And thus we just return the actual instance count. if (count == null) { return serviceManagementMap.get(serviceName).getInstances(); } else { return count; } }
@Override public void containerLaunched(String runnableName, int instanceId, String containerId) { super.containerLaunched(runnableName, instanceId, containerId); if (runningPublished.compareAndSet(false, true)) { // The program is marked as running when the first container for the program is launched programStateWriterWithHeartBeat.running(twillRunId.getId()); } }
@Override public boolean accept(String className, URL classUrl, URL classPathUrl) { // Exclude both hadoop and spark classes. if (urls.contains(classPathUrl)) { return false; } return super.accept(className, classUrl, classPathUrl) && !className.startsWith("org.apache.spark."); } };
private Map<String, String> getMetricContext(TwillRunner.LiveInfo info) { ProgramId programId = TwillAppNames.fromTwillAppName(info.getApplicationName(), false); if (programId == null) { return null; } return getMetricsContext(programId.getType(), programId); } }
@Override public void completed() { super.completed(); // On normal AM completion, based on the last container failure to publish the state if (lastContainerFailure == null) { programStateWriterWithHeartBeat.completed(); } else { lastContainerFailure.writeError(programStateWriterWithHeartBeat); } }
@Override public void containerStopped(String runnableName, int instanceId, String containerId, int exitStatus) { super.containerStopped(runnableName, instanceId, containerId, exitStatus); // Let the completed() method handle when a container has completed with no error if (exitStatus == 0) { return; } switch(programRunId.getType()) { case WORKFLOW: case SPARK: case MAPREDUCE: // For workflow, MapReduce, and spark, if there is an error, the program state is failure // We defer the actual publish to one of the completion methods (killed, completed, aborted) // as we need to know under what condition the container failed. lastContainerFailure = new ContainerFailure(runnableName, instanceId, containerId, exitStatus); break; default: // For other programs, the container will be re-launched - the program state will continue to be RUNNING // TODO Workers should be configured via runtime args // to support both retrying on failure, or just failing and not retrying. break; } }
@Override protected void configure() { bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class); } }
@Override public void killed() { super.killed(); // The AM is stopped explicitly, always record the state as killed. programStateWriterWithHeartBeat.killed(); }
/** * Creates the {@link EventHandler} for handling the application events. */ @Nullable private EventHandler createEventHandler(CConfiguration cConf, ProgramRunId programRunId) { if (clusterMode != ClusterMode.ON_PREMISE) { return null; } return new TwillAppLifecycleEventHandler(cConf.getLong(Constants.CFG_TWILL_NO_CONTAINER_TIMEOUT, Long.MAX_VALUE), this instanceof LongRunningDistributedProgramRunner, programRunId); }
private int getSystemServiceInstanceCount(String serviceName) throws Exception { Integer count = serviceStore.getServiceInstance(serviceName); //In standalone mode, this count will be null. And thus we just return the actual instance count. if (count == null) { return serviceManagementMap.get(serviceName).getInstances(); } else { return count; } }
@Override public boolean accept(String className, URL classUrl, URL classPathUrl) { // Exclude both hadoop and spark classes. if (urls.contains(classPathUrl)) { return false; } return super.accept(className, classUrl, classPathUrl) && !className.startsWith("org.apache.spark."); } };
@Override public boolean accept(String className, URL classUrl, URL classPathUrl) { if (className.startsWith("org.apache.spark") || classPathUrl.toString().contains("spark-assembly")) { return false; } return hadoopClassExcluder.accept(className, classUrl, classPathUrl); } });
@Override public boolean accept(String className, URL classUrl, URL classPathUrl) { if (className.startsWith("org.apache.spark") || classPathUrl.toString().contains("spark-assembly")) { return false; } return hadoopClassExcluder.accept(className, classUrl, classPathUrl); } });