dag.addStream("Kinesis message", getOutputPortOfGenerator( generator ), node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
StreamConf streamConf = streamConfEntry.getValue(); DAG.StreamMeta sd = dag.addStream(streamConfEntry.getKey()); sd.setLocality(streamConf.getLocality()); Operators.PortMappingDescriptor sourcePortMap = new Operators.PortMappingDescriptor(); Operators.describe(sourceDecl, sourcePortMap); sd.setSource(sourcePortMap.outputPorts.get(portName).component); Operators.PortMappingDescriptor targetPortMap = new Operators.PortMappingDescriptor(); Operators.describe(targetDecl, targetPortMap); sd.addSink(targetPortMap.inputPorts.get(portName).component);
@Test public void testParDoChaining() throws Exception { Pipeline p = Pipeline.create(); long numElements = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements)); PAssert.thatSingleton(input.apply("Count", Count.globally())).isEqualTo(numElements); ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class); DAG dag = TestApexRunner.translate(p, options); String[] expectedThreadLocal = {"/CreateActual/FilterActuals/Window.Assign"}; Set<String> actualThreadLocal = Sets.newHashSet(); for (DAG.StreamMeta sm : dag.getAllStreamsMeta()) { DAG.OutputPortMeta opm = sm.getSource(); if (sm.getLocality() == Locality.THREAD_LOCAL) { String name = opm.getOperatorMeta().getName(); String prefix = "PAssert$"; if (name.startsWith(prefix)) { // remove indeterministic prefix name = name.substring(prefix.length() + 1); } actualThreadLocal.add(name); } } Assert.assertThat(actualThreadLocal, Matchers.hasItems(expectedThreadLocal)); } }
.setLocality(Locality.CONTAINER_LOCAL);
.setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("Kafka message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
generator.setSyncUrl("tcp://localhost:5557"); dag.addStream("Stream", generator.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL); new Thread()
collector.setSUBSCRIBERS_EXPECTED(1); dag.addStream("Stream", source.outPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
publisher.setup(); dag.addStream("Stream", consumer.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("Kafka message", generator.outputPort, node.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL);
dag.addStream("Kafka message", generator.outputPort, node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("Kafka message", generator.outputPort, node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("transactions", txReceiver.txOutputPort, txBucketOperator.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL); dag.addStream("txData", txReceiver.txDataOutputPort, hdfsOutputOperator.input); // dump all tx into Hdfs dag.addStream("userTransactions", txInputHandler.txOutputPort, txBucketOperator.txUserInputPort);
dag.addStream("Kinesis message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("queue1", generator.outputPort, hbaseOutputOperator.input ).setLocality(DAG.Locality.NODE_LOCAL); dag.addStream("queue2", hbaseInputOperator.outputPort, output.inputPort ).setLocality(DAG.Locality.NODE_LOCAL);
dag.addStream("Kinesis Records", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("Kafka message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("Kinesis stream", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
streamMeta.setLocality(entry.getValue().getRight());
dag.addStream("TweetStream", twitterFeed.status, HashtagExtractor.input).setLocality(Locality.CONTAINER_LOCAL); dag.addStream("SendToKinesis", HashtagExtractor.hashtags, outputOp.inputPort).setLocality(locality); topCounts.setDagWindowWidth(1); dag.addStream("TwittedHashtags", inputOp.outputPort, uniqueCounter.data).setLocality(locality); dag.addStream("UniqueHashtagCounts", uniqueCounter.count, topCounts.input).setLocality(locality); dag.addStream("TopHashtags", topCounts.output, consoleOutput(dag, "topHashtags")).setLocality(locality);