/** * Instantiate {@link org.apache.apex.malhar.lib.math.SumKeyVal} operator * Get aggregated volume of 1 minute and send at the end window of 1 minute. * @param name Operator name * @param dag Application DAG graph. * @param appWindowCount Operator aggregate window size. * @return SumKeyVal instance. */ public SumKeyVal<String, Long> getMinuteVolumeOperator(String name, DAG dag, int appWindowCount) { SumKeyVal<String, Long> oper = dag.addOperator(name, new SumKeyVal<String, Long>()); oper.setType(Long.class); dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount); return oper; }
/** * Instantiate {@link org.apache.apex.malhar.lib.math.RangeKeyVal} operator to get high/low * value for each key within given application window. * Get High-low range for 1 minute. * @param name Operator name * @param dag Application DAG graph. * @param appWindowCount Operator aggregate window size. * @return RangeKeyVal instance. */ public RangeKeyVal<String, Double> getHighLowOperator(String name, DAG dag, int appWindowCount) { RangeKeyVal<String, Double> oper = dag.addOperator(name, new RangeKeyVal<String, Double>()); dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount); oper.setType(Double.class); return oper; }
@Test public void testConfigProperties() throws Exception { String operName = "testProperties"; ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class); // default configuration from class path Pipeline p = Pipeline.create(); Create.Values<Void> empty = Create.empty(VoidCoder.of()); p.apply(operName, empty); DAG dag = TestApexRunner.translate(p, options); OperatorMeta t1Meta = dag.getOperatorMeta(operName); Assert.assertNotNull(t1Meta); Assert.assertEquals(Integer.valueOf(32), t1Meta.getValue(OperatorContext.MEMORY_MB)); File tmp = File.createTempFile("beam-runners-apex-", ".properties"); tmp.deleteOnExit(); Properties props = new Properties(); props.setProperty("apex.operator." + operName + ".attr.MEMORY_MB", "64"); try (FileOutputStream fos = new FileOutputStream(tmp)) { props.store(fos, ""); } options.setConfigFile(tmp.getAbsolutePath()); dag = TestApexRunner.translate(p, options); tmp.delete(); t1Meta = dag.getOperatorMeta(operName); Assert.assertNotNull(t1Meta); Assert.assertEquals(Integer.valueOf(64), t1Meta.getValue(OperatorContext.MEMORY_MB)); }
@Test public void test() throws Exception { ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class); EmbeddedCollector.RESULTS.clear(); options.setApplicationName("ReadUnbound"); options.setRunner(ApexRunner.class); Pipeline p = Pipeline.create(options); List<String> collection = Lists.newArrayList("1", "2", "3", "4", "5"); CollectionSource<String> source = new CollectionSource<>(collection, StringUtf8Coder.of()); p.apply(Read.from(source)).apply(ParDo.of(new EmbeddedCollector())); ApexRunnerResult result = (ApexRunnerResult) p.run(); DAG dag = result.getApexDAG(); DAG.OperatorMeta om = dag.getOperatorMeta("Read(CollectionSource)"); Assert.assertNotNull(om); Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class); long timeout = System.currentTimeMillis() + 30000; while (System.currentTimeMillis() < timeout) { if (EmbeddedCollector.RESULTS.containsAll(collection)) { break; } LOG.info("Waiting for expected results."); Thread.sleep(1000); } Assert.assertEquals(Sets.newHashSet(collection), EmbeddedCollector.RESULTS); }
@Test public void testReadBounded() throws Exception { ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class); EmbeddedCollector.RESULTS.clear(); options.setApplicationName("ReadBounded"); options.setRunner(ApexRunner.class); Pipeline p = Pipeline.create(options); Set<Long> expected = ContiguousSet.create(Range.closedOpen(0L, 10L), DiscreteDomain.longs()); p.apply(GenerateSequence.from(0).to(10)).apply(ParDo.of(new EmbeddedCollector())); ApexRunnerResult result = (ApexRunnerResult) p.run(); DAG dag = result.getApexDAG(); String operatorName = "GenerateSequence/Read(BoundedCountingSource)"; DAG.OperatorMeta om = dag.getOperatorMeta(operatorName); Assert.assertNotNull(om); Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class); long timeout = System.currentTimeMillis() + 30000; while (System.currentTimeMillis() < timeout) { if (EmbeddedCollector.RESULTS.containsAll(expected)) { break; } LOG.info("Waiting for expected results."); Thread.sleep(1000); } Assert.assertEquals(Sets.newHashSet(expected), EmbeddedCollector.RESULTS); }
@Test public void test() throws Exception { ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class); options.setApplicationName("ParDoBound"); options.setRunner(ApexRunner.class); Pipeline p = Pipeline.create(options); List<Integer> collection = Lists.newArrayList(1, 2, 3, 4, 5); List<Integer> expected = Lists.newArrayList(6, 7, 8, 9, 10); p.apply(Create.of(collection).withCoder(SerializableCoder.of(Integer.class))) .apply(ParDo.of(new Add(5))) .apply(ParDo.of(new EmbeddedCollector())); ApexRunnerResult result = (ApexRunnerResult) p.run(); DAG dag = result.getApexDAG(); DAG.OperatorMeta om = dag.getOperatorMeta("Create.Values"); Assert.assertNotNull(om); Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class); om = dag.getOperatorMeta("ParDo(Add)/ParMultiDo(Add)"); Assert.assertNotNull(om); Assert.assertEquals(om.getOperator().getClass(), ApexParDoOperator.class); long timeout = System.currentTimeMillis() + TIMEOUT_MILLIS; while (System.currentTimeMillis() < timeout) { if (EmbeddedCollector.RESULTS.containsAll(expected)) { break; } LOG.info("Waiting for expected results."); Thread.sleep(SLEEP_MILLIS); } Assert.assertEquals(Sets.newHashSet(expected), EmbeddedCollector.RESULTS); }
@Test public void testFlattenSingleCollection() { ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class); Pipeline p = Pipeline.create(); PCollection<String> single = p.apply(Create.of(Collections.singletonList("1"))); PCollectionList.of(single) .apply(Flatten.pCollections()) .apply(ParDo.of(new EmbeddedCollector())); DAG dag = TestApexRunner.translate(p, options); Assert.assertNotNull( dag.getOperatorMeta("ParDo(EmbeddedCollector)/ParMultiDo(EmbeddedCollector)")); } }
Assert.assertNotNull(result.getApexDAG().getOperatorMeta("Read(UnboundedTextSource)")); long timeout = System.currentTimeMillis() + 30000; while (System.currentTimeMillis() < timeout) {
TestApexRunner.translate(p, PipelineOptionsFactory.create().as(ApexPipelineOptions.class)); OperatorMeta om = dag.getOperatorMeta("ParDo/ParMultiDo(Anonymous)"); assertNotNull(om); assertEquals(2, om.getInputStreams().size());