public SimpleProgramOptions(ProgramId programId) { this(programId, new BasicArguments(), new BasicArguments()); }
public SimpleProgramOptions(ProgramId programId) { this(programId, new BasicArguments(), new BasicArguments()); }
@Override public Arguments deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { Map<String, String> args = context.deserialize(json, MAP_STRING_STRING_TYPE); return new BasicArguments(args); }
@Override public Arguments deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { Map<String, String> args = context.deserialize(json, MAP_STRING_STRING_TYPE); return new BasicArguments(args); }
private ProgramOptions createProgramOptions(ProgramId programId, Map<String, String> properties) { String userArgumentsString = properties.get(ProgramOptionConstants.USER_OVERRIDES); String systemArgumentsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES); String debugString = properties.get(ProgramOptionConstants.DEBUG_ENABLED); Boolean debug = Boolean.valueOf(debugString); Map<String, String> userArguments = userArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(userArgumentsString, STRING_STRING_MAP); Map<String, String> systemArguments = systemArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(systemArgumentsString, STRING_STRING_MAP); return new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments(userArguments), debug); }
private ProgramOptions createProgramOptions(ProgramId programId, Map<String, String> properties) { String userArgumentsString = properties.get(ProgramOptionConstants.USER_OVERRIDES); String systemArgumentsString = properties.get(ProgramOptionConstants.SYSTEM_OVERRIDES); String debugString = properties.get(ProgramOptionConstants.DEBUG_ENABLED); Boolean debug = Boolean.valueOf(debugString); Map<String, String> userArguments = userArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(userArgumentsString, STRING_STRING_MAP); Map<String, String> systemArguments = systemArgumentsString == null ? Collections.emptyMap() : GSON.fromJson(systemArgumentsString, STRING_STRING_MAP); return new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments(userArguments), debug); }
private ProgramOptions resolveFlowletOptions(ProgramOptions options, String flowlet) { Map<String, String> systemArgs = new HashMap<>(options.getArguments().asMap()); systemArgs.put(ProgramOptionConstants.FLOWLET_NAME, flowlet); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArgs), new BasicArguments(RuntimeArguments.extractScope( FlowUtils.FLOWLET_SCOPE, flowlet, options.getUserArguments().asMap()))); }
private ProgramOptions createFlowletOptions(int instanceId, int instances, ProgramOptions options) { Map<String, String> systemArgs = new HashMap<>(); systemArgs.putAll(options.getArguments().asMap()); systemArgs.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemArgs.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArgs), options.getUserArguments()); }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
@Override protected Arguments resolveScope(Arguments arguments) { return new BasicArguments(RuntimeArguments.extractScope(FlowUtils.FLOWLET_SCOPE, name, arguments.asMap())); }
/** * Creates a {@link ProgramOptions} by deserializing the given json file. */ private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException { ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class); // Overwrite them with environmental information Map<String, String> arguments = new HashMap<>(original.getArguments().asMap()); arguments.putAll(getExtraSystemArguments()); // Use the name passed in by the constructor as the program name to construct the ProgramId return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), resolveScope(original.getUserArguments()), original.isDebug()); }
/** * Creates a {@link ProgramOptions} by deserializing the given json file. */ private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException { ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class); // Overwrite them with environmental information Map<String, String> arguments = new HashMap<>(original.getArguments().asMap()); arguments.putAll(getExtraSystemArguments()); // Use the name passed in by the constructor as the program name to construct the ProgramId return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), resolveScope(original.getUserArguments()), original.isDebug()); }
private void runProgram(ApplicationWithPrograms app, Class<?> programClass, Map<String, String> userArgs, boolean expectedStatus) throws Exception { LOG.info("Starting {} with arguments {}", programClass.getName(), userArgs); Assert.assertEquals(expectedStatus, runProgram(app, programClass, new BasicArguments(userArgs))); }
/** * Tests that initialize() and getSplits() are called in the same transaction, * and with the same instance of the input dataset. */ @Test public void testTransactionHandling() throws Exception { final ApplicationWithPrograms app = deployApp(AppWithTxAware.class); runProgram(app, AppWithTxAware.PedanticMapReduce.class, new BasicArguments(ImmutableMap.of("outputPath", TEMP_FOLDER_SUPPLIER.get().getPath() + "/output"))); }
private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate()); Map<String, String> systemArgs = new HashMap<>(); Map<String, String> userArgs = new HashMap<>(); Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo); SystemArguments.addProfileArgs(systemArgs, profile); ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs)); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec); return new TaskFields(programDescriptor, programOptions, programRunId); }
@Test public void testMapperOutputTypeChecking() throws Exception { final ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingInconsistentMappers.class); // the mapreduce with consistent mapper types will succeed Assert.assertTrue(runProgram(app, AppWithMapReduceUsingInconsistentMappers.MapReduceWithConsistentMapperTypes.class, new BasicArguments())); // the mapreduce with mapper classes of inconsistent output types will fail, whether the mappers are set through // CDAP APIs or also directly on the job Assert.assertFalse(runProgram(app, AppWithMapReduceUsingInconsistentMappers.MapReduceWithInconsistentMapperTypes.class, new BasicArguments())); Assert.assertFalse(runProgram(app, AppWithMapReduceUsingInconsistentMappers.MapReduceWithInconsistentMapperTypes2.class, new BasicArguments())); }
@Test public void testConfigIsolation() throws Exception { ApplicationWithPrograms app = deployApp(AppWithSingleInputOutput.class); Assert.assertTrue(runProgram(app, AppWithSingleInputOutput.SimpleMapReduce.class, new BasicArguments())); } }
@Test public void testAddingMultipleOutputsWithSameAlias() throws Exception { final ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingMultipleOutputs.class); // will fail because it configured two outputs with the same alias Assert.assertFalse(runProgram(app, AppWithMapReduceUsingMultipleOutputs.InvalidMapReduce.class, new BasicArguments())); } }
@Test public void testAddingMultipleInputsWithSameAlias() throws Exception { final ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingMultipleInputs.class); // will fail because it configured two inputs with the same alias Assert.assertFalse(runProgram(app, AppWithMapReduceUsingMultipleInputs.InvalidMapReduce.class, new BasicArguments())); } }