if( getGroup() != null ) conf.setNumReduceTasks( numSinkParts ); else conf.setNumMapTasks( numSinkParts ); else if( getGroup() != null ) if( getGroup() == null ) if( getGroup().isSortReversed() ) conf.setOutputKeyComparatorClass( ReverseTupleComparator.class ); addComparators( conf, "cascading.group.comparator", getGroup().getKeySelectors(), (Fields) processEdge.getResolvedKeyFields().get( ordinal ) ); if( getGroup().isGroupBy() ) addComparators( conf, "cascading.sort.comparator", getGroup().getSortingSelectors(), (Fields) processEdge.getResolvedSortFields().get( ordinal ) ); if( !getGroup().isGroupBy() ) if( getGroup().isSorted() ) if( getGroup().isSortReversed() ) conf.setOutputKeyComparatorClass( ReverseGroupingSortingComparator.class ); else
if( getGroup() != null ) conf.setNumReduceTasks( numSinkParts ); else conf.setNumMapTasks( numSinkParts ); else if( getGroup() != null ) if( getGroup() == null ) if( getGroup().isSortReversed() ) conf.setOutputKeyComparatorClass( ReverseTupleComparator.class ); addComparators( conf, "cascading.group.comparator", getGroup().getKeySelectors(), (Fields) processEdge.getResolvedKeyFields().get( ordinal ) ); if( getGroup().isGroupBy() ) addComparators( conf, "cascading.sort.comparator", getGroup().getSortingSelectors(), (Fields) processEdge.getResolvedSortFields().get( ordinal ) ); if( !getGroup().isGroupBy() ) if( getGroup().isSorted() ) if( getGroup().isSortReversed() ) conf.setOutputKeyComparatorClass( ReverseGroupingSortingComparator.class ); else
/** * Test a single piece Pipe, should not fail, inserts Identity pipe * * @throws IOException */ @Test public void testIdentity() throws Exception { Tap source = new Hfs( new TextLine(), "input/path" ); Tap sink = new Hfs( new TextLine(), "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); List<FlowStep> steps = flow.getFlowSteps(); assertEquals( "wrong size", 1, steps.size() ); HadoopFlowStep step = (HadoopFlowStep) steps.get( 0 ); assertEquals( "not equal: step.sources.size()", 1, step.getSourceTaps().size() ); assertNull( "not null: step.groupBy", step.getGroup() ); assertNotNull( "null: step.sink", step.getSink() ); }
/** * Test a single piece Pipe, should not fail, inserts Identity pipe * * @throws IOException */ @Test public void testIdentity() throws Exception { Tap source = new Hfs( new TextLine(), "input/path" ); Tap sink = new Hfs( new TextLine(), "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); List<FlowStep> steps = flow.getFlowSteps(); assertEquals( "wrong size", 1, steps.size() ); HadoopFlowStep step = (HadoopFlowStep) steps.get( 0 ); assertEquals( "not equal: step.sources.size()", 1, step.getSourceTaps().size() ); assertNull( "not null: step.groupBy", step.getGroup() ); assertNotNull( "null: step.sink", step.getSink() ); }