@Override public ProgramManager apply(ApplicationManager input) { return input.getMapReduceManager(AppWithPartitionConsumers.WordCountMapReduce.NAME).start(); } }, true);
@Test public void testAdminMapReduce() throws Exception { testAdminBatchProgram(appManager.getMapReduceManager(AdminApp.MAPREDUCE_NAME)); }
@Test public void testCrossNSMapReduce() throws Exception { createAuthNamespace(); ApplicationId appId = AUTH_NAMESPACE.app(DatasetCrossNSAccessWithMAPApp.class.getSimpleName()); Map<EntityId, Set<Action>> neededPrivileges = ImmutableMap.<EntityId, Set<Action>>builder() .put(appId, EnumSet.of(Action.ADMIN)) .put(AUTH_NAMESPACE.artifact(DatasetCrossNSAccessWithMAPApp.class.getSimpleName(), "1.0-SNAPSHOT"), EnumSet.of(Action.ADMIN)) .build(); setUpPrivilegeAndRegisterForDeletion(ALICE, neededPrivileges); ProgramId programId = appId.program(ProgramType.MAPREDUCE, DatasetCrossNSAccessWithMAPApp.MAPREDUCE_PROGRAM); // bob will be executing the program grantAndAssertSuccess(programId, BOB, EnumSet.of(Action.EXECUTE)); cleanUpEntities.add(programId); ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, DatasetCrossNSAccessWithMAPApp.class); MapReduceManager mrManager = appManager.getMapReduceManager(DatasetCrossNSAccessWithMAPApp.MAPREDUCE_PROGRAM); testCrossNSSystemDatasetAccessWithAuthMapReduce(mrManager); testCrossNSDatasetAccessWithAuthMapReduce(mrManager); }
@Category(SlowTests.class) @Test public void testMapReduceTaskMetricsDisable() throws Exception { addDatasetInstance("keyValueTable", "table1"); addDatasetInstance("keyValueTable", "table2"); DataSetManager<KeyValueTable> tableManager = getDataset("table1"); KeyValueTable inputTable = tableManager.get(); inputTable.write("hello", "world"); tableManager.flush(); ApplicationManager appManager = deployApplication(DatasetWithMRApp.class); Map<String, String> argsForMR = ImmutableMap.of(DatasetWithMRApp.INPUT_KEY, "table1", DatasetWithMRApp.OUTPUT_KEY, "table2", "task.*." + SystemArguments.METRICS_ENABLED, "false"); MapReduceManager mrManager = appManager.getMapReduceManager(DatasetWithMRApp.MAPREDUCE_PROGRAM).start(argsForMR); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); appManager.stopAll(); testTaskMetric(mrManager.getHistory().get(0).getPid(), false); }
args.putAll(RuntimeArguments.addScope(Scope.DATASET, ClusterNameTestApp.OUTPUT_FILE_SET, outputArgs)); MapReduceManager mrManager = appManager.getMapReduceManager( ClusterNameTestApp.ClusterNameMapReduce.class.getSimpleName()).start(args); key.set("mr.client.cluster.name");
@Category(SlowTests.class) @Test public void testMapperDatasetAccess() throws Exception { addDatasetInstance("keyValueTable", "table1"); addDatasetInstance("keyValueTable", "table2"); DataSetManager<KeyValueTable> tableManager = getDataset("table1"); KeyValueTable inputTable = tableManager.get(); inputTable.write("hello", "world"); tableManager.flush(); ApplicationManager appManager = deployApplication(DatasetWithMRApp.class); Map<String, String> argsForMR = ImmutableMap.of(DatasetWithMRApp.INPUT_KEY, "table1", DatasetWithMRApp.OUTPUT_KEY, "table2", "task.*." + SystemArguments.METRICS_CONTEXT_TASK_INCLUDED, "false"); MapReduceManager mrManager = appManager.getMapReduceManager(DatasetWithMRApp.MAPREDUCE_PROGRAM).start(argsForMR); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); appManager.stopAll(); DataSetManager<KeyValueTable> outTableManager = getDataset("table2"); verifyMapperJobOutput(DatasetWithMRApp.class, outTableManager); // test that the metrics emitted by MR task is collected testTaskMetric(mrManager.getHistory().get(0).getPid(), true); // test that the metrics is not emitted with instance tag(task-level) testTaskTagLevelExists(DatasetWithMRApp.class.getSimpleName(), DatasetWithMRApp.MAPREDUCE_PROGRAM, mrManager.getHistory().get(0).getPid(), "table1", false); }
MapReduceManager mrManager = appManager.getMapReduceManager("DynamicPartitioningMR"); int numRuns = mrManager.getHistory(ProgramRunStatus.FAILED).size(); mrManager.start(arguments);
@Test public void test() throws Exception { ApplicationManager applicationManager = deployApplication(TestMapReduceServiceIntegrationApp.class); ServiceManager serviceManager = applicationManager.getServiceManager(TestMapReduceServiceIntegrationApp.SERVICE_NAME).start(); serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS); DataSetManager<MyKeyValueTableDefinition.KeyValueTable> inDataSet = getDataset(TestMapReduceServiceIntegrationApp.INPUT_DATASET); inDataSet.get().write("key1", "Two words"); inDataSet.get().write("key2", "Plus three words"); inDataSet.flush(); MapReduceManager mrManager = applicationManager.getMapReduceManager(TestMapReduceServiceIntegrationApp.MR_NAME).start(); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 180, TimeUnit.SECONDS); DataSetManager<MyKeyValueTableDefinition.KeyValueTable> outDataSet = getDataset(TestMapReduceServiceIntegrationApp.OUTPUT_DATASET); MyKeyValueTableDefinition.KeyValueTable results = outDataSet.get(); String total = results.get(TestMapReduceServiceIntegrationApp.SQUARED_TOTAL_WORDS_COUNT); Assert.assertEquals(25, Integer.parseInt(total)); } }
@Category(SlowTests.class) @Test public void testCrossNSMapperDatasetAccess() throws Exception { NamespaceMeta inputNS = new NamespaceMeta.Builder().setName("inputNS").build(); NamespaceMeta outputNS = new NamespaceMeta.Builder().setName("outputNS").build(); getNamespaceAdmin().create(inputNS); getNamespaceAdmin().create(outputNS); addDatasetInstance(inputNS.getNamespaceId().dataset("table1"), "keyValueTable"); addDatasetInstance(outputNS.getNamespaceId().dataset("table2"), "keyValueTable"); DataSetManager<KeyValueTable> tableManager = getDataset(inputNS.getNamespaceId().dataset("table1")); KeyValueTable inputTable = tableManager.get(); inputTable.write("hello", "world"); tableManager.flush(); ApplicationManager appManager = deployApplication(DatasetCrossNSAccessWithMAPApp.class); Map<String, String> argsForMR = ImmutableMap.of( DatasetCrossNSAccessWithMAPApp.INPUT_DATASET_NS, inputNS.getName(), DatasetCrossNSAccessWithMAPApp.INPUT_DATASET_NAME, "table1", DatasetCrossNSAccessWithMAPApp.OUTPUT_DATASET_NS, outputNS.getName(), DatasetCrossNSAccessWithMAPApp.OUTPUT_DATASET_NAME, "table2"); MapReduceManager mrManager = appManager.getMapReduceManager(DatasetCrossNSAccessWithMAPApp.MAPREDUCE_PROGRAM) .start(argsForMR); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); appManager.stopAll(); DataSetManager<KeyValueTable> outTableManager = getDataset(outputNS.getNamespaceId().dataset("table2")); verifyMapperJobOutput(DatasetCrossNSAccessWithMAPApp.class, outTableManager); }
@Test public void testPFSRollback() throws Exception { ApplicationManager appManager = deployApplication(AppWritingToPartitioned.class); MapReduceManager mrManager = appManager.getMapReduceManager(MAPREDUCE); int numRuns = 0;
.start(txTimeoutArguments(txDefaulTimeoutWorkflow, txDefaulTimeoutAction, "action", AppWithCustomTx.ACTION_NOTX)); MapReduceManager txMRManager = appManager.getMapReduceManager(AppWithCustomTx.MAPREDUCE_TX) .start(txTimeoutArguments(txDefaulTimeoutMapReduce)); MapReduceManager notxMRManager = appManager.getMapReduceManager(AppWithCustomTx.MAPREDUCE_NOTX) .start(txTimeoutArguments(txDefaulTimeoutMapReduce)); SparkManager txSparkManager = appManager.getSparkManager(AppWithCustomTx.SPARK_TX)