/** * Retrieves and updates ambrose node associated with the given flow step. * * @param step step with which to update ambrose node state. * @return node associated with step. */ private DAGNode<CascadingJob> updateNode(FlowStep step) { DAGNode<CascadingJob> node = getNode(step); CascadingJob job = node.getJob(); HadoopStepStats stats = (HadoopStepStats) step.getFlowStepStats(); job.setId(stats.getJobID()); job.setJobStats(stats); mapReduceHelper.addMapReduceJobState(job, stats.getJobClient()); return node; }
/** * Retrieves the ambrose node associated with the given flow step. * * @param step step for which node should be retrieved. * @return node associated with step. */ private DAGNode<CascadingJob> getNode(FlowStep step) { String name = step.getName(); DAGNode<CascadingJob> node = nodesByName.get(name); if (node == null) { throw new IllegalStateException(String.format("Node with name '%s' not found", name)); } return node; }
private Configuration getConfig() { return (Configuration) this.getFlowStep().getConfig(); }
Configuration config = (Configuration) flowStep.getConfig(); Iterator<FlowNode> iterator = flowStep.getFlowNodeGraph().getOrderedTopologicalIterator();
protected void initSteps() { if( flowStepGraph == null ) return; Set<FlowStep> flowSteps = flowStepGraph.vertexSet(); for( FlowStep flowStep : flowSteps ) { ( (BaseFlowStep) flowStep ).setFlow( this ); Set<FlowNode> flowNodes = flowStep.getFlowNodeGraph().vertexSet(); for( FlowNode flowNode : flowNodes ) ( (BaseFlowNode) flowNode ).setFlowStep( flowStep ); } }
return; int stepOrdinal = flowStep.getOrdinal(); Path rootPath = path.resolve( directoryName ); ElementGraph stepSubGraph = flowStep.getElementGraph(); String stepGraphName = String.format( "%s/%04d-step-sub-graph-%s.dot", rootPath, stepOrdinal, canonicalHash( stepSubGraph ) ); FlowNodeGraph flowNodeGraph = flowStep.getFlowNodeGraph();
protected FlowStepStats( FlowStep flowStep, ClientState clientState ) { super( flowStep.getName(), clientState ); this.flowStep = flowStep; this.flowStep.setFlowStepStats( this ); }
Tap sink = step.getSink(); if( flow.getSink() != sink && sink.resourceExists( flow.getConfig() ) ) count++;
private void prepareBloomFilterBuilder(FlowStep<JobConf> currentStep) { JobConf currentStepConf = currentStep.getConfig(); currentStepConf.set("mapred.reduce.tasks", Integer.toString(BloomProps.getNumSplits(currentStepConf))); }
Configuration config = (Configuration) flowStep.getConfig(); Iterator<FlowNode> iterator = flowStep.getFlowNodeGraph().getOrderedTopologicalIterator();
List<ElementGraph> elementGraphs = flowStep.getFlowNodeGraph().getElementGraphs( hashJoin );
Tap sink = step.getSink(); if( flow.getSink() != sink && sink.resourceExists( flow.getConfig() ) ) count++;
private Configuration getConfig() { return (Configuration) this.getFlowStep().getConfig(); }
/** * Retrieves and updates ambrose node associated with the given flow step. * * @param step step with which to update ambrose node state. * @return node associated with step. */ private DAGNode<CascadingJob> updateNode(FlowStep step) { DAGNode<CascadingJob> node = getNode(step); CascadingJob job = node.getJob(); HadoopStepStats stats = (HadoopStepStats) step.getFlowStepStats(); job.setId(stats.getProcessStepID()); job.setJobStats(stats); mapReduceHelper.addMapReduceJobState(job, stats.getJobClient()); return node; }
List<ElementGraph> elementGraphs = flowStep.getFlowNodeGraph().getElementGraphs( hashJoin );
/** * Retrieves the ambrose node associated with the given flow step. * * @param step step for which node should be retrieved. * @return node associated with step. */ private DAGNode<CascadingJob> getNode(FlowStep step) { String name = step.getName(); DAGNode<CascadingJob> node = nodesByName.get(name); if (node == null) { throw new IllegalStateException(String.format("Node with name '%s' not found", name)); } return node; }
for( FlowStep step : steps ) if( step.getSink() instanceof DecoratorTap ) count++;
@Override public void apply(Flow<JobConf> flow, List<FlowStep<JobConf>> predecessorSteps, FlowStep<JobConf> flowStep) { JobConf conf = flowStep.getConfig(); String targetBloomID = conf.get(BloomProps.TARGET_BLOOM_FILTER_ID); if (targetBloomID != null) { prepareBloomFilterBuilder(flowStep); } // the job is the filter which needs to use the bloom filter String sourceBloomID = conf.get(BloomProps.SOURCE_BLOOM_FILTER_ID); if (sourceBloomID != null) { buildBloomfilter(sourceBloomID, flowStep, predecessorSteps); } }
@Override public void onStepStopping(FlowStep flowStep) { LOG.info("Step stopping"); HadoopStepStats hdStepStats = (HadoopStepStats)flowStep.getFlowStepStats(); recordStepData(hdStepStats); }
/** * onStepProgressing event is fired whenever a job makes progress. * * @param step the step in the flow that represents the MapReduce job. */ @Override public void onStepRunning(FlowStep step) { // first we report the scripts progress int progress = (int) ((((double) runningJobs) / totalNumberOfJobs) * 100); AmbroseUtils.pushWorkflowProgressEvent(statsWriteService, currentFlowId, progress); // only push job progress events for a completed step once if (completedStepNames.contains(step.getName())) { return; } try { // update node DAGNode<CascadingJob> node = updateNode(step); if (node.getJob().getMapReduceJobState() != null) { AmbroseUtils.pushEvent(statsWriteService, currentFlowId, new Event.JobProgressEvent(node)); if (node.getJob().getMapReduceJobState().isComplete()) { completedStepNames.add(step.getName()); } } } catch (Exception e) { LOG.error("Failed to handle onStepRunning event", e); } }