@Override public ProgramManager apply(ApplicationManager input) { return input.getMapReduceManager(AppWithPartitionConsumers.WordCountMapReduce.NAME).start(); } }, true);
ClusterNameTestApp.ClusterNameMapReduce.class.getSimpleName()).start(args); key.set("mr.client.cluster.name"); Tasks.waitFor(clusterName, readClusterName, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
@Category(SlowTests.class) @Test public void testMapReduceTaskMetricsDisable() throws Exception { addDatasetInstance("keyValueTable", "table1"); addDatasetInstance("keyValueTable", "table2"); DataSetManager<KeyValueTable> tableManager = getDataset("table1"); KeyValueTable inputTable = tableManager.get(); inputTable.write("hello", "world"); tableManager.flush(); ApplicationManager appManager = deployApplication(DatasetWithMRApp.class); Map<String, String> argsForMR = ImmutableMap.of(DatasetWithMRApp.INPUT_KEY, "table1", DatasetWithMRApp.OUTPUT_KEY, "table2", "task.*." + SystemArguments.METRICS_ENABLED, "false"); MapReduceManager mrManager = appManager.getMapReduceManager(DatasetWithMRApp.MAPREDUCE_PROGRAM).start(argsForMR); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); appManager.stopAll(); testTaskMetric(mrManager.getHistory().get(0).getPid(), false); }
@Category(SlowTests.class) @Test public void testMapperDatasetAccess() throws Exception { addDatasetInstance("keyValueTable", "table1"); addDatasetInstance("keyValueTable", "table2"); DataSetManager<KeyValueTable> tableManager = getDataset("table1"); KeyValueTable inputTable = tableManager.get(); inputTable.write("hello", "world"); tableManager.flush(); ApplicationManager appManager = deployApplication(DatasetWithMRApp.class); Map<String, String> argsForMR = ImmutableMap.of(DatasetWithMRApp.INPUT_KEY, "table1", DatasetWithMRApp.OUTPUT_KEY, "table2", "task.*." + SystemArguments.METRICS_CONTEXT_TASK_INCLUDED, "false"); MapReduceManager mrManager = appManager.getMapReduceManager(DatasetWithMRApp.MAPREDUCE_PROGRAM).start(argsForMR); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); appManager.stopAll(); DataSetManager<KeyValueTable> outTableManager = getDataset("table2"); verifyMapperJobOutput(DatasetWithMRApp.class, outTableManager); // test that the metrics emitted by MR task is collected testTaskMetric(mrManager.getHistory().get(0).getPid(), true); // test that the metrics is not emitted with instance tag(task-level) testTaskTagLevelExists(DatasetWithMRApp.class.getSimpleName(), DatasetWithMRApp.MAPREDUCE_PROGRAM, mrManager.getHistory().get(0).getPid(), "table1", false); }
MapReduceManager mrManager = appManager.getMapReduceManager("DynamicPartitioningMR"); int numRuns = mrManager.getHistory(ProgramRunStatus.FAILED).size(); mrManager.start(arguments); Tasks.waitFor(numRuns + 1, () -> mrManager.getHistory(ProgramRunStatus.FAILED).size(), 300, TimeUnit.SECONDS);
mrManager.start(argsForMR); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
@Test public void test() throws Exception { ApplicationManager applicationManager = deployApplication(TestMapReduceServiceIntegrationApp.class); ServiceManager serviceManager = applicationManager.getServiceManager(TestMapReduceServiceIntegrationApp.SERVICE_NAME).start(); serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS); DataSetManager<MyKeyValueTableDefinition.KeyValueTable> inDataSet = getDataset(TestMapReduceServiceIntegrationApp.INPUT_DATASET); inDataSet.get().write("key1", "Two words"); inDataSet.get().write("key2", "Plus three words"); inDataSet.flush(); MapReduceManager mrManager = applicationManager.getMapReduceManager(TestMapReduceServiceIntegrationApp.MR_NAME).start(); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 180, TimeUnit.SECONDS); DataSetManager<MyKeyValueTableDefinition.KeyValueTable> outDataSet = getDataset(TestMapReduceServiceIntegrationApp.OUTPUT_DATASET); MyKeyValueTableDefinition.KeyValueTable results = outDataSet.get(); String total = results.get(TestMapReduceServiceIntegrationApp.SQUARED_TOTAL_WORDS_COUNT); Assert.assertEquals(25, Integer.parseInt(total)); } }
mrManager.start(ImmutableMap.of(PFS_OUT, "1", "input.text", "1x")); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); mrManager.start(ImmutableMap.of(PFS_OUT, "2", "input.text", "2x")); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); mrManager.start(ImmutableMap.of(PFS_OUT, "4", "input.text", "4x")); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); mrManager.start(ImmutableMap.of("input.text", "3x 5x")); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); mrManager.start(ImmutableMap.of("input.text", "2x 5x")); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); mrManager.start(ImmutableMap.of("input.text", "0x 4x 5x")); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); mrManager.start(ImmutableMap.of("output.datasets", BOTH, PFS_OUT, "1", "input.text", "0x 5x")); mrManager.waitForRuns(ProgramRunStatus.FAILED, ++numRuns, 2, TimeUnit.MINUTES); pfsValidator.validate(); mrManager.start(ImmutableMap.of("output.datasets", BOTH, OTHER_OUT, "1", "input.text", "0x 5x"));
@Category(SlowTests.class) @Test public void testCrossNSMapperDatasetAccess() throws Exception { NamespaceMeta inputNS = new NamespaceMeta.Builder().setName("inputNS").build(); NamespaceMeta outputNS = new NamespaceMeta.Builder().setName("outputNS").build(); getNamespaceAdmin().create(inputNS); getNamespaceAdmin().create(outputNS); addDatasetInstance(inputNS.getNamespaceId().dataset("table1"), "keyValueTable"); addDatasetInstance(outputNS.getNamespaceId().dataset("table2"), "keyValueTable"); DataSetManager<KeyValueTable> tableManager = getDataset(inputNS.getNamespaceId().dataset("table1")); KeyValueTable inputTable = tableManager.get(); inputTable.write("hello", "world"); tableManager.flush(); ApplicationManager appManager = deployApplication(DatasetCrossNSAccessWithMAPApp.class); Map<String, String> argsForMR = ImmutableMap.of( DatasetCrossNSAccessWithMAPApp.INPUT_DATASET_NS, inputNS.getName(), DatasetCrossNSAccessWithMAPApp.INPUT_DATASET_NAME, "table1", DatasetCrossNSAccessWithMAPApp.OUTPUT_DATASET_NS, outputNS.getName(), DatasetCrossNSAccessWithMAPApp.OUTPUT_DATASET_NAME, "table2"); MapReduceManager mrManager = appManager.getMapReduceManager(DatasetCrossNSAccessWithMAPApp.MAPREDUCE_PROGRAM) .start(argsForMR); mrManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); appManager.stopAll(); DataSetManager<KeyValueTable> outTableManager = getDataset(outputNS.getNamespaceId().dataset("table2")); verifyMapperJobOutput(DatasetCrossNSAccessWithMAPApp.class, outTableManager); }
txDefaulTimeoutAction, "action", AppWithCustomTx.ACTION_NOTX)); MapReduceManager txMRManager = appManager.getMapReduceManager(AppWithCustomTx.MAPREDUCE_TX) .start(txTimeoutArguments(txDefaulTimeoutMapReduce)); MapReduceManager notxMRManager = appManager.getMapReduceManager(AppWithCustomTx.MAPREDUCE_NOTX) .start(txTimeoutArguments(txDefaulTimeoutMapReduce)); SparkManager txSparkManager = appManager.getSparkManager(AppWithCustomTx.SPARK_TX) .start(txTimeoutArguments(txDefaulTimeoutSpark));