/** * Method retrieveSinkFields notifies a Scheme when it is appropriate to dynamically * update the fields it sources. By default the current declared fields are returned. * <p> * The {@code FlowProcess} presents all known properties resolved by the current planner. * <p> * The {@code tap} instance is the parent {@link Tap} for this Scheme instance. * * @param flowProcess of type FlowProcess * @param tap of type Tap * @return Fields */ public Fields retrieveSinkFields( FlowProcess<? extends Config> flowProcess, Tap tap ) { return getSinkFields(); }
public Fields getSinkFields() { if( partitionFields == null || scheme.getSinkFields().isAll() ) return scheme.getSinkFields(); return Fields.merge( scheme.getSinkFields(), partitionFields ); }
/** * Method getSinkFields returns the sinkFields of this Tap object. * * @return the sinkFields (type Fields) of this Tap object. */ public Fields getSinkFields() { return getScheme().getSinkFields(); }
public LocalScheme(Scheme<JobConf, RecordReader, OutputCollector, SourceContext, SinkContext> scheme) { super(scheme.getSourceFields(), scheme.getSinkFields()); this.scheme = scheme; }
protected void presentSinkFieldsInternal( Fields fields ) { if( getSinkFields().equals( Fields.ALL ) ) setSinkFields( fields ); }
@Override public String toString() { if( getSinkFields().equals( getSourceFields() ) ) return getClass().getSimpleName() + "[" + getSourceFields().print() + "]"; else return getClass().getSimpleName() + "[" + getSourceFields().print() + "->" + getSinkFields().print() + "]"; }
/** * Method isSymmetrical returns {@code true} if the sink fields equal the source fields. That is, this * scheme sources the same fields as it sinks. * * @return the symmetrical (type boolean) of this Scheme object. */ public boolean isSymmetrical() { return getSourceFields().equals( Fields.UNKNOWN ) && getSinkFields().equals( Fields.ALL ) || getSinkFields().equals( getSourceFields() ); }
public TupleEntrySchemeCollector( FlowProcess<? extends Config> flowProcess, Tap tap, Scheme scheme, Output output, Supplier<String> loggableIdentifier ) { super( Fields.asDeclaration( scheme.getSinkFields() ) ); this.flowProcess = flowProcess; this.scheme = scheme; if( loggableIdentifier != null ) this.loggableIdentifier = loggableIdentifier; // only used for logging this.sinkCall = createSinkCall(); this.sinkCall.setTap( tap ); this.sinkCall.setOutgoingEntry( this.tupleEntry ); // created in super ctor if( output != null ) setOutput( output ); }
private void setFields( Scheme scheme ) { if( scheme.isSource() ) { Fields sourceFields = normalize( scheme.getSourceFields() ); if( fields == null ) fields = sourceFields; else if( !fields.equals( sourceFields ) ) throw new IllegalArgumentException( "all schemes added to stereotype must have the same source fields, expected: " + fields + ", received: " + sourceFields + " in stereotype: " + getName() ); } if( scheme.isSink() ) { Fields sinkFields = normalize( scheme.getSinkFields() ); if( fields == null ) fields = sinkFields; else if( !fields.equals( sinkFields ) ) throw new IllegalArgumentException( "all schemes added to stereotype must have the same sink fields, expected: " + fields + ", received: " + sinkFields + " in stereotype: " + getName() ); } }
protected void verifyCheckpoints( FlowDef flowDef, Pipe[] flowTails ) { verifyNotSourcesSinks( flowDef.getCheckpoints(), flowDef.getSources(), flowDef.getSinks(), "checkpoint" ); for( Tap checkpointTap : flowDef.getCheckpoints().values() ) { Scheme scheme = checkpointTap.getScheme(); if( scheme.getSourceFields().equals( Fields.UNKNOWN ) && scheme.getSinkFields().equals( Fields.ALL ) ) continue; throw new PlannerException( "checkpoint tap scheme must be undeclared, source fields must be UNKNOWN, and sink fields ALL, got: " + scheme.toString() ); } Set<String> names = new HashSet<String>( asList( Pipe.names( flowTails ) ) ); for( String name : flowDef.getCheckpoints().keySet() ) { if( !names.contains( name ) ) throw new PlannerException( "named checkpoint declared in FlowDef, but no named branch found in pipe assembly: '" + name + "'" ); Set<Pipe> pipes = new HashSet<Pipe>( asList( Pipe.named( name, flowTails ) ) ); int count = 0; for( Pipe pipe : pipes ) { if( pipe instanceof Checkpoint ) count++; } if( count == 0 ) throw new PlannerException( "no checkpoint pipe with branch name found in pipe assembly: '" + name + "'" ); if( count > 1 ) throw new PlannerException( "more than one checkpoint pipe with branch name found in pipe assembly: '" + name + "'" ); } }