@Override public ProgramOptions deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { JsonObject jsonObj = json.getAsJsonObject(); ProgramId programId = context.deserialize(jsonObj.get("programId"), ProgramId.class); Arguments arguments = context.deserialize(jsonObj.get("arguments"), Arguments.class); Arguments userArguments = context.deserialize(jsonObj.get("userArguments"), Arguments.class); boolean debug = jsonObj.get("debug").getAsBoolean(); return new SimpleProgramOptions(programId, arguments, userArguments, debug); }
@Override public ProgramOptions deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { JsonObject jsonObj = json.getAsJsonObject(); ProgramId programId = context.deserialize(jsonObj.get("programId"), ProgramId.class); Arguments arguments = context.deserialize(jsonObj.get("arguments"), Arguments.class); Arguments userArguments = context.deserialize(jsonObj.get("userArguments"), Arguments.class); boolean debug = jsonObj.get("debug").getAsBoolean(); return new SimpleProgramOptions(programId, arguments, userArguments, debug); }
private ProgramOptions createProgramOptions(ProgramId programId, Map<String, String> properties) { String userArgumentsString = properties.get(ProgramOptionConstants.USER_OVERRIDES); String systemArgumentsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES); String debugString = properties.get(ProgramOptionConstants.DEBUG_ENABLED); Boolean debug = Boolean.valueOf(debugString); Map<String, String> userArguments = userArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(userArgumentsString, STRING_STRING_MAP); Map<String, String> systemArguments = systemArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(systemArgumentsString, STRING_STRING_MAP); return new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments(userArguments), debug); }
private ProgramOptions createProgramOptions(ProgramId programId, Map<String, String> properties) { String userArgumentsString = properties.get(ProgramOptionConstants.USER_OVERRIDES); String systemArgumentsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES); String debugString = properties.get(ProgramOptionConstants.DEBUG_ENABLED); Boolean debug = Boolean.valueOf(debugString); Map<String, String> userArguments = userArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(userArgumentsString, STRING_STRING_MAP); Map<String, String> systemArguments = systemArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(systemArgumentsString, STRING_STRING_MAP); return new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments(userArguments), debug); }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
private ProgramOptions createFlowletOptions(int instanceId, int instances, ProgramOptions options) { Map<String, String> systemArgs = new HashMap<>(); systemArgs.putAll(options.getArguments().asMap()); systemArgs.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemArgs.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArgs), options.getUserArguments()); }
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
/** * Creates a {@link ProgramOptions} by deserializing the given json file. */ private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException { ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class); // Overwrite them with environmental information Map<String, String> arguments = new HashMap<>(original.getArguments().asMap()); arguments.putAll(getExtraSystemArguments()); // Use the name passed in by the constructor as the program name to construct the ProgramId return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), resolveScope(original.getUserArguments()), original.isDebug()); }
/** * Creates a {@link ProgramOptions} by deserializing the given json file. */ private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException { ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class); // Overwrite them with environmental information Map<String, String> arguments = new HashMap<>(original.getArguments().asMap()); arguments.putAll(getExtraSystemArguments()); // Use the name passed in by the constructor as the program name to construct the ProgramId return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), resolveScope(original.getUserArguments()), original.isDebug()); }
private ProgramOptions resolveFlowletOptions(ProgramOptions options, String flowlet) { Map<String, String> systemArgs = new HashMap<>(options.getArguments().asMap()); systemArgs.put(ProgramOptionConstants.FLOWLET_NAME, flowlet); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArgs), new BasicArguments(RuntimeArguments.extractScope( FlowUtils.FLOWLET_SCOPE, flowlet, options.getUserArguments().asMap()))); }
return new SimpleProgramOptions(programId, new BasicArguments(systemArgs), new BasicArguments(userArgs), debug);
private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate()); Map<String, String> systemArgs = new HashMap<>(); Map<String, String> userArgs = new HashMap<>(); Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo); SystemArguments.addProfileArgs(systemArgs, profile); ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs)); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec); return new TaskFields(programDescriptor, programOptions, programRunId); }
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").program(ProgramType.SPARK, "spark").run(RunIds.generate().getId()); ProgramOptions programOpts = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(Collections.singletonMap("a", "b" + i)), new BasicArguments(Collections.singletonMap("x", "y" + i)));
private void publishProgramStatus(ProgramRunId programRunId, ProgramStatus status) { ProgramStateWriter stateWriter = new MessagingProgramStateWriter(cConf, messagingService); switch (status) { case INITIALIZING: stateWriter.start(programRunId, new SimpleProgramOptions(programRunId.getParent()), null, null); break; case RUNNING: stateWriter.running(programRunId, null); break; case COMPLETED: stateWriter.completed(programRunId); break; case FAILED: stateWriter.error(programRunId, new Exception("Program run failed")); break; case KILLED: stateWriter.killed(programRunId); break; default: throw new IllegalArgumentException("Unsupported program status " + status); } }
@Test public void testAppSpecNotRequiredToWriteState() throws Exception { ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s"); Map<String, String> systemArguments = new HashMap<>(); systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()); systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()); ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments()); ProgramRunId runId = programId.run(RunIds.generate()); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec); programStateWriter.start(runId, programOptions, null, programDescriptor); Tasks.waitFor(ProgramRunStatus.STARTING, () -> txnl.execute(() -> { RunRecordMeta meta = metadataStoreDataset.getRun(runId); if (meta == null) { return null; } Assert.assertEquals(artifactId, meta.getArtifactId()); return meta.getStatus(); }), 10, TimeUnit.SECONDS); programStateWriter.completed(runId); }
/** * Submits a program execution. * * @param app the application containing the program * @param programClassName name of the program class * @param userArgs runtime arguments * @param folderSupplier a Supplier of temporary folder * @return a {@link ProgramController} for controlling the program execution. */ public static ProgramController submit(ApplicationWithPrograms app, String programClassName, Arguments userArgs, Supplier<File> folderSupplier) throws Exception { ProgramRunnerFactory runnerFactory = injector.getInstance(ProgramRunnerFactory.class); ProgramRunner runner = null; Program program = null; for (ProgramDescriptor programDescriptor : app.getPrograms()) { if (programDescriptor.getSpecification().getClassName().equals(programClassName)) { runner = runnerFactory.create(programDescriptor.getProgramId().getType()); program = createProgram(programDescriptor, app.getArtifactLocation(), runner, folderSupplier); break; } } Assert.assertNotNull(program); BasicArguments systemArgs = new BasicArguments(ImmutableMap.of( ProgramOptionConstants.RUN_ID, RunIds.generate().getId(), ProgramOptionConstants.HOST, InetAddress.getLoopbackAddress().getCanonicalHostName(), ProgramOptionConstants.ARTIFACT_ID, Joiner.on(":").join(app.getArtifactId().toIdParts()) )); return runner.run(program, new SimpleProgramOptions(program.getId(), systemArgs, userArgs)); }
@Test public void testReadWrite() { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").program(ProgramType.SPARK, "spark").run(RunIds.generate().getId()); ProgramOptions programOpts = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(Collections.singletonMap("a", "b")), new BasicArguments(Collections.singletonMap("x", "y"))); // Write the state Transactionals.execute(transactional, context -> { RemoteRuntimeDataset dataset = RemoteRuntimeDataset.create(context, datasetFramework); dataset.write(programRunId, programOpts); }); // Use scan to read it back Optional<Map.Entry<ProgramRunId, ProgramOptions>> result = Transactionals.execute(transactional, context -> { RemoteRuntimeDataset dataset = RemoteRuntimeDataset.create(context, datasetFramework); return dataset.scan(1, null).stream().findFirst(); }); Map.Entry<ProgramRunId, ProgramOptions> entry = result.orElseThrow(IllegalStateException::new); Assert.assertEquals(programRunId, entry.getKey()); Assert.assertTrue(programOptionsEquals(programOpts, entry.getValue())); }
@Test public void testDelete() { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").program(ProgramType.SPARK, "spark").run(RunIds.generate().getId()); ProgramOptions programOpts = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(Collections.singletonMap("a", "b")), new BasicArguments(Collections.singletonMap("x", "y"))); // Write a task info Transactionals.execute(transactional, context -> { RemoteRuntimeDataset dataset = RemoteRuntimeDataset.create(context, datasetFramework); dataset.write(programRunId, programOpts); }); // Delete the task info Transactionals.execute(transactional, context -> { RemoteRuntimeDataset dataset = RemoteRuntimeDataset.create(context, datasetFramework); dataset.delete(programRunId); }); // Scan should be empty boolean result = Transactionals.execute(transactional, context -> { RemoteRuntimeDataset dataset = RemoteRuntimeDataset.create(context, datasetFramework); return dataset.scan(1, null).isEmpty(); }); Assert.assertTrue(result); }