@Override public void populateDAG(DAG dag, Configuration conf) { Gen gen = dag.addOperator("gen", Gen.class); DevNull devNull = dag.addOperator("devNull", DevNull.class); dag.addStream("data", gen.out, devNull.data); } }
@Override public void populateDAG(DAG dag, Configuration conf) { // Creating an example application with three operators // The last operator is slowing down the DAG // With the use of the stats listener the input operator is slowed when the window difference crosses a threshold RandomNumberGenerator randomGenerator = dag.addOperator("RandomGenerator", RandomNumberGenerator.class); PassThroughOperator<Double> passThrough = dag.addOperator("PassThrough", PassThroughOperator.class); SlowDevNullOperator<Double> devNull = dag.addOperator("SlowNull", SlowDevNullOperator.class); // Important to use the same stats listener object for all operators so that we can centrally collect stats and make // the decision StatsListener statsListener = new ThrottlingStatsListener(); Collection<StatsListener> statsListeners = Lists.newArrayList(statsListener); dag.setAttribute(randomGenerator, Context.OperatorContext.STATS_LISTENERS, statsListeners); dag.setAttribute(passThrough, Context.OperatorContext.STATS_LISTENERS, statsListeners); dag.setAttribute(devNull, Context.OperatorContext.STATS_LISTENERS, statsListeners); // Increase timeout for the slow operator, this specifies the maximum timeout for an operator to process a window // It is specified in number of windows, since 1 window is 500ms, 30 minutes is 30 * 60 * 2 = 3600 windows dag.setAttribute(devNull, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600); // If there are unifiers that are slow then set timeout for them // dag.setUnifierAttribute(passThrough.output, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600); dag.addStream("randomData", randomGenerator.out, passThrough.input); dag.addStream("passData", passThrough.output, devNull.input); } }
@Override public void populateDAG(DAG dag, Configuration conf) { RandomNumberGenerator random = dag.addOperator("randomInt", RandomNumberGenerator.class); TestPartition testPartition = dag.addOperator("testPartition", TestPartition.class); Codec3 codec = new Codec3(); dag.setInputPortAttribute(testPartition.in, PortContext.STREAM_CODEC, codec); //Add locality if needed, e.g.: .setLocality(Locality.CONTAINER_LOCAL); dag.addStream("randomData", random.out, testPartition.in); } }
public void populateDAG(DAG dag, Configuration conf) { FSRecordReaderModule recordReader = dag.addModule("lineInput", FSRecordReaderModule.class); S3BytesOutputModule s3StringOutputModule = dag.addModule("s3TupleOutput", S3BytesOutputModule.class); dag.addStream("data", recordReader.records, s3StringOutputModule.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { ParquetFilePOJOReader parquetReader = dag.addOperator("parquetReader", getParquetFilePOJOReader()); ConsoleOutputOperator pojoOp = dag.addOperator("pojoOp", new ConsoleOutputOperator()); dag.getMeta(parquetReader).getMeta(parquetReader.output).getAttributes().put(Context.PortContext.TUPLE_CLASS, EventRecord.class); dag.addStream("pojo", parquetReader.output, pojoOp.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class); CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class); CsvFormatter formatter = dag.addOperator("formatter", new CsvFormatter()); StringFileOutputOperator fileOutput = dag.addOperator("fileOutput", new StringFileOutputOperator()); dag.addStream("record", recordReader.records, csvParser.in); dag.addStream("pojo", csvParser.out, formatter.in); dag.addStream("string", formatter.out, fileOutput.input); } }
@Override public void populateDAG(DAG dag, Configuration conf) { pojoDataGenerator = dag.addOperator("Input", new POJOGenerator()); TransformOperator transform = dag.addOperator("Process", new TransformOperator()); // Set expression map Map<String, String> expMap = new HashMap<>(); expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})"); expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()"); expMap.put("address", "{$.address}.toLowerCase()"); transform.setExpressionMap(expMap); FunctionOperator.MapFunctionOperator<Object, ?> output = dag.addOperator("out", new FunctionOperator.MapFunctionOperator<>(outputFn)); dag.addStream("InputToTransform", pojoDataGenerator.output, transform.input); dag.addStream("TransformToOutput", transform.output, output.input); dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class); dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class); setPartitioner(dag,conf,transform); }
public ConsoleOutputOperator getConsoleOperator(String name, DAG dag, String prefix, String format) { ConsoleOutputOperator oper = dag.addOperator(name, ConsoleOutputOperator.class); // oper.setStringFormat(prefix + ": " + format); return oper; }
S3InitiateFileUploadOperator initiateUpload = dag.addOperator("InitiateUpload", createS3InitiateUpload()); initiateUpload.setAccessKey(accessKey); initiateUpload.setSecretAccessKey(secretAccessKey); initiateUpload.setOutputDirectoryPath(outputDirectoryPath); S3BlockUploadOperator blockUploader = dag.addOperator("BlockUpload", createS3BlockUpload()); blockUploader.setAccessKey(accessKey); blockUploader.setSecretAccessKey(secretAccessKey); blockUploader.setBucketName(bucketName); S3FileMerger fileMerger = dag.addOperator("FileMerger", createS3FileMerger()); fileMerger.setAccessKey(accessKey); fileMerger.setSecretAccessKey(secretAccessKey); dag.setInputPortAttribute(blockUploader.blockInput, Context.PortContext.PARTITION_PARALLEL, true); dag.setInputPortAttribute(blockUploader.blockMetadataInput, Context.PortContext.PARTITION_PARALLEL, true); dag.setAttribute(initiateUpload, TIMEOUT_WINDOW_COUNT, timeOutWindowCount); dag.setAttribute(blockUploader, TIMEOUT_WINDOW_COUNT, timeOutWindowCount); dag.setAttribute(fileMerger, TIMEOUT_WINDOW_COUNT, timeOutWindowCount); dag.setUnifierAttribute(blockUploader.output, TIMEOUT_WINDOW_COUNT, timeOutWindowCount); dag.setAttribute(fileMerger,Context.OperatorContext.PARTITIONER, new StatelessPartitioner<S3FileMerger>(mergerCount)); dag.addStream("InitiateUploadIDToMerger", initiateUpload.fileMetadataOutput, fileMerger.filesMetadataInput); dag.addStream("InitiateUploadIDToWriter", initiateUpload.uploadMetadataOutput, blockUploader.uploadMetadataInput); dag.addStream("WriterToMerger", blockUploader.output, fileMerger.uploadMetadataInput);
@Override public void populateDAG(DAG dag, Configuration configuration) { AvroFileInputOperator avroFileInputOperator = dag.addOperator("AvroFileInputOperator", this.avroFileInputOperator); AvroToPojo avroToPojo = dag.addOperator("AvroGenericObjectToPojo", new AvroToPojo()); dag.setOutputPortAttribute(avroToPojo.output, Context.PortContext.TUPLE_CLASS, pojoClass); dag.addStream("avroFileContainerToPojo", avroFileInputOperator.output, avroToPojo.data) .setLocality(DAG.Locality.CONTAINER_LOCAL); output.set(avroToPojo.output); errorPort.set(avroToPojo.errorPort); completedAvroFilesPort.set(avroFileInputOperator.completedFilesPort); avroErrorRecordsPort.set(avroFileInputOperator.errorRecordsPort); }
@Override public void populateDAG(DAG dag, Configuration conf) { locality = Locality.CONTAINER_LOCAL; dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000); SimpleSinglePortZeroMQPullStringInputOperator input = dag.addOperator("input", new SimpleSinglePortZeroMQPullStringInputOperator(addr)); ApacheLogParseOperator parse = dag.addOperator("parse", new ApacheLogParseOperator()); UniqueCounter<String> ipAddrCount = dag.addOperator("ipAddrCount", new UniqueCounter<String>()); UniqueCounter<String> urlCount = dag.addOperator("urlCount", new UniqueCounter<String>()); UniqueCounter<String> httpStatusCount = dag.addOperator("httpStatusCount", new UniqueCounter<String>()); Sum<Long> numOfBytesSum = dag.addOperator("numOfBytesSum", new Sum<Long>()); //ArrayListAggregator<Long> agg = dag.addOperator("agg", new ArrayListAggregator<Long>()); //dag.getOperatorWrapper(agg).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, 3); dag.getMeta(numOfBytesSum).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, 3); dag.addStream("input-parse", input.outputPort, parse.data).setLocality(locality); dag.addStream("parse-ipAddrCount", parse.outputIPAddress, ipAddrCount.data).setLocality(locality); dag.addStream("parse-urlCount", parse.outputUrl, urlCount.data).setLocality(locality); dag.addStream("parse-httpStatusCount", parse.outputStatusCode, httpStatusCount.data).setLocality(locality); dag.addStream("parse-numOfBytesSum", parse.outputBytes, numOfBytesSum.data).setLocality(locality); //dag.addStream("numOfBytesSum-agg", numOfBytesSum.sumLong, agg.input); ConsoleOutputOperator consoleOperator1 = dag.addOperator("console1", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator2 = dag.addOperator("console2", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator3 = dag.addOperator("console3", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator4 = dag.addOperator("console4", new ConsoleOutputOperator()); dag.addStream("ipAddrCount-console", ipAddrCount.count, consoleOperator1.input); dag.addStream("urlCount-console", urlCount.count, consoleOperator2.input); dag.addStream("httpStatusCount-console", httpStatusCount.count, consoleOperator3.input); //dag.addStream("agg-console", agg.output, consoleOperator4.input); dag.addStream("numOfBytesSum-console", numOfBytesSum.sumLong, consoleOperator4.input); }
/** * Populate Yahoo Finance Example Application DAG. */ @SuppressWarnings("unchecked") @Override public void populateDAG(DAG dag, Configuration conf) { dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds); StockTickInput tick = getStockTickInputOperator("StockTickInput", dag); SumKeyVal<String, Long> dailyVolume = getDailyVolumeOperator("DailyVolume", dag); ConsolidatorKeyVal<String,Double,Long,String,?,?> quoteOperator = getQuoteOperator("Quote", dag); RangeKeyVal<String, Double> highlow = getHighLowOperator("HighLow", dag, appWindowCountMinute); SumKeyVal<String, Long> minuteVolume = getMinuteVolumeOperator("MinuteVolume", dag, appWindowCountMinute); ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> chartOperator = getChartOperator("Chart", dag); SimpleMovingAverage<String, Double> priceSMA = getPriceSimpleMovingAverageOperator("PriceSMA", dag, appWindowCountSMA); DefaultPartitionCodec<String, Double> codec = new DefaultPartitionCodec<String, Double>(); dag.setInputPortAttribute(highlow.data, PortContext.STREAM_CODEC, codec); dag.setInputPortAttribute(priceSMA.data, PortContext.STREAM_CODEC, codec); dag.addStream("price", tick.price, quoteOperator.in1, highlow.data, priceSMA.data); dag.addStream("vol", tick.volume, dailyVolume.data, minuteVolume.data); dag.addStream("time", tick.time, quoteOperator.in3); dag.addStream("daily_vol", dailyVolume.sum, quoteOperator.in2); dag.addStream("quote_data", quoteOperator.out, getConsole("quoteConsole", dag, "QUOTE")); dag.addStream("high_low", highlow.range, chartOperator.in1); dag.addStream("vol_1min", minuteVolume.sum, chartOperator.in2); dag.addStream("chart_data", chartOperator.out, getConsole("chartConsole", dag, "CHART")); dag.addStream("sma_price", priceSMA.doubleSMA, getConsole("priceSMAConsole", dag, "Price SMA")); }
/** * Instantiate {@link org.apache.apex.malhar.lib.math.SumKeyVal} operator * Get aggregated volume of 1 minute and send at the end window of 1 minute. * @param name Operator name * @param dag Application DAG graph. * @param appWindowCount Operator aggregate window size. * @return SumKeyVal instance. */ public SumKeyVal<String, Long> getMinuteVolumeOperator(String name, DAG dag, int appWindowCount) { SumKeyVal<String, Long> oper = dag.addOperator(name, new SumKeyVal<String, Long>()); oper.setType(Long.class); dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount); return oper; }
void setPartitioner(DAG dag,Configuration conf, TransformOperator transform) { dag.setAttribute(transform, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TransformOperator>(2)); } }
@Override public void populateDAG(DAG dag, Configuration conf) { AvroFileInputOperator avroInputOperator = dag.addOperator("avroInputOperator", getAvroFileInput()); AvroToPojo avroToPojo = dag.addOperator("AvroToPojo", getAvroToPojo()); ConsoleOutputOperator consoleOutput = dag.addOperator("GenericRecordOp", new ConsoleOutputOperator()); dag.getMeta(avroToPojo).getMeta(avroToPojo.output).getAttributes().put(Context.PortContext.TUPLE_CLASS, SimpleOrder.class); dag.addStream("GenericRecords", avroInputOperator.output, avroToPojo.data).setLocality(Locality.THREAD_LOCAL); dag.addStream("POJO", avroToPojo.output, consoleOutput.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class); CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class); FilterOperator filterOperator = dag.addOperator("filterOperator", new FilterOperator()); CsvFormatter selectedFormatter = dag.addOperator("selectedFormatter", new CsvFormatter()); CsvFormatter rejectedFormatter = dag.addOperator("rejectedFormatter", new CsvFormatter()); StringFileOutputOperator selectedOutput = dag.addOperator("selectedOutput", new StringFileOutputOperator()); StringFileOutputOperator rejectedOutput = dag.addOperator("rejectedOutput", new StringFileOutputOperator()); dag.addStream("record", recordReader.records, csvParser.in); dag.addStream("pojo", csvParser.out, filterOperator.input); dag.addStream("pojoSelected", filterOperator.truePort, selectedFormatter.in); dag.addStream("pojoRejected", filterOperator.falsePort, rejectedFormatter.in); dag.addStream("csvSelected", selectedFormatter.out, selectedOutput.input); dag.addStream("csvRejected", rejectedFormatter.out, rejectedOutput.input); } }
dag.addOperator(opName, nm.operator); DAG.StreamMeta streamMeta = dag.addStream(entry.getKey().toString(), entry.getKey(), entry.getValue().getLeft().toArray(new Operator.InputPort[]{})); for (Pair<Attribute, Object> attr : nm.outputPortAttributes.get(entry.getKey())) { logger.debug("Building DAG: set port attribute {} to {} for port {}", attr.getLeft(), attr.getValue(), entry.getKey()); dag.setOutputPortAttribute(entry.getKey(), attr.getLeft(), attr.getValue()); for (Pair<Attribute, Object> attr : nm.inputPortAttributes.get(input)) { logger.debug("Building DAG: set port attribute {} to {} for port {}", attr.getLeft(), attr.getValue(), input); dag.setInputPortAttribute(input, attr.getLeft(), attr.getValue()); dag.setAttribute(nm.operator, attr.getLeft(), attr.getValue());
/** * Instantiate stock input operator for actual Yahoo finance ticks of symbol, last price, total daily volume, and last traded price. * @param name Operator name * @param dag Application DAG graph. * @return StockTickInput instance. */ public StockTickInput getStockTickInputOperator(String name, DAG dag) { StockTickInput oper = dag.addOperator(name, StockTickInput.class); oper.readIntervalMillis = 200; //oper.symbols = tickers; return oper; }
@Override public void populateDAG(DAG dag, Configuration conf) { FSInputModule inputModule = dag.addModule("HDFSInputModule", new FSInputModule()); S3OutputModule outputModule = dag.addModule("S3OutputModule", new S3OutputModule()); dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput); dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput) .setLocality(DAG.Locality.CONTAINER_LOCAL); dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(DAG.Locality.CONTAINER_LOCAL); } }
/** * Instantiate {@link org.apache.apex.malhar.lib.math.RangeKeyVal} operator to get high/low * value for each key within given application window. * Get High-low range for 1 minute. * @param name Operator name * @param dag Application DAG graph. * @param appWindowCount Operator aggregate window size. * @return RangeKeyVal instance. */ public RangeKeyVal<String, Double> getHighLowOperator(String name, DAG dag, int appWindowCount) { RangeKeyVal<String, Double> oper = dag.addOperator(name, new RangeKeyVal<String, Double>()); dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount); oper.setType(Double.class); return oper; }