private Set<FlowElement> getSinks(FlowNode node) { return node.getSinkElements(); }
private Scope getOutScope(FlowNode node) { Set<FlowElement> nodeSinks = node.getSinkElements(); if(nodeSinks.size() != 1) { throw new RuntimeException("Only nodes with one output supported right now"); } FlowElement sink = nodeSinks.iterator().next(); Collection<Scope> outScopes = (Collection<Scope>) node.getPreviousScopes(sink); if(outScopes.size() != 1) { throw new RuntimeException("Only one incoming scope for last node of mapper allowed"); } return outScopes.iterator().next(); }
protected Map<FlowElement, Configuration> initFromSinks( FlowNode flowNode, FlowProcess<? extends Configuration> flowProcess, Configuration conf ) { Set<FlowElement> sinks = flowNode.getSinkElements(); Map<FlowElement, Configuration> configs = new HashMap<>(); for( FlowElement element : sinks ) { JobConf current = new JobConf( conf ); if( element instanceof Tap ) { Tap tap = (Tap) element; if( tap.getIdentifier() == null ) throw new IllegalStateException( "tap may not have null identifier: " + element.toString() ); tap.sinkConfInit( flowProcess, current ); setLocalMode( conf, current, tap ); } String id = FlowElements.id( element ); current.set( "cascading.node.sink", id ); configs.put( element, current ); } return configs; }
protected Map<FlowElement, Configuration> initFromSinks( FlowNode flowNode, FlowProcess<? extends Configuration> flowProcess, Configuration conf ) { Set<FlowElement> sinks = flowNode.getSinkElements(); Map<FlowElement, Configuration> configs = new HashMap<>(); for( FlowElement element : sinks ) { JobConf current = new JobConf( conf ); if( element instanceof Tap ) { Tap tap = (Tap) element; if( tap.getIdentifier() == null ) throw new IllegalStateException( "tap may not have null identifier: " + element.toString() ); tap.sinkConfInit( flowProcess, current ); setLocalMode( conf, current, tap ); } String id = FlowElements.id( element ); current.set( "cascading.node.sink", id ); configs.put( element, current ); } return configs; }
private DataSet<?> prepareCoGroupInput(List<DataSet<Tuple>> inputs, FlowNode node, int dop) { CoGroup coGroup = (CoGroup)getSingle(node.getSinkElements()); Joiner joiner = coGroup.getJoiner(); int numJoinInputs = coGroup.isSelfJoin() ? coGroup.getNumSelfJoins() + 1 : inputs.size(); Fields[] inputFields = new Fields[numJoinInputs]; Fields[] keyFields = new Fields[numJoinInputs]; String[][] flinkKeys = new String[numJoinInputs][]; List<DataSet<Tuple>> joinInputs = computeSpliceInputsFieldsKeys(coGroup, node, inputs, inputFields, keyFields, flinkKeys); if(joiner.getClass().equals(InnerJoin.class)) { if(!keyFields[0].isNone()) { return prepareFullOuterCoGroupInput(joinInputs, node, inputFields, keyFields, flinkKeys, dop); } else { // Cartesian product return prepareInnerCrossInput(joinInputs, node, inputFields, dop); } } else if(joiner.getClass().equals(BufferJoin.class)) { return prepareBufferCoGroupInput(joinInputs, node, inputFields, keyFields, flinkKeys, dop); } else { return prepareFullOuterCoGroupInput(joinInputs, node, inputFields, keyFields, flinkKeys, dop); } }