/** * Method setSinkFields sets the sinkFields of this Scheme object. * * @param sinkFields the sinkFields of this Scheme object. */ public void setSinkFields( Fields sinkFields ) { if( sinkFields.isUnknown() ) this.sinkFields = Fields.ALL; else this.sinkFields = sinkFields; }
/** * Constructor RegexSplitter creates a new RegexSplitter instance. * * @param patternString of type String */ @ConstructorProperties({"patternString"}) public RegexSplitter( String patternString ) { super( 1, patternString ); length = fieldDeclaration.isUnknown() ? -1 : fieldDeclaration.size(); }
/** * Constructor RegexSplitter creates a new RegexSplitter instance. * * @param fieldDeclaration of type Fields * @param patternString of type String */ @ConstructorProperties({"fieldDeclaration", "patternString"}) public RegexSplitter( Fields fieldDeclaration, String patternString ) { super( 1, fieldDeclaration, patternString ); length = fieldDeclaration.isUnknown() ? -1 : fieldDeclaration.size(); }
final int translatePos( Integer integer, int size ) { if( size == -1 ) return integer; if( integer < 0 ) integer = size + integer; if( !isUnknown() && ( integer >= size || integer < 0 ) ) throw new TupleException( "position value is too large: " + integer + ", positions in field: " + size ); return integer; }
/** * Constructor RegexParser creates a new RegexParser instance, where the patternString is a regular expression * with match groups and whose groups designated by {@code groups} are stored in the named fieldDeclarations. * * @param fieldDeclaration of type Fields * @param patternString of type String * @param groups of type int[] */ @ConstructorProperties({"fieldDeclaration", "patternString", "groups"}) public RegexParser( Fields fieldDeclaration, String patternString, int... groups ) { super( 1, fieldDeclaration, patternString ); verifyReturnLength( groups ); this.groups = Arrays.copyOf( groups, groups.length ); if( !fieldDeclaration.isUnknown() && fieldDeclaration.size() != groups.length ) throw new IllegalArgumentException( "fieldDeclaration must equal number of groups to be captured, fields: " + fieldDeclaration.print() ); }
/** * Method getPos returns the pos array of this Fields object. * * @return the pos (type int[]) of this Fields object. */ public int[] getPos() { if( thisPos != null ) return thisPos; // do not clone if( isAll() || isUnknown() ) thisPos = EMPTY_INT; else thisPos = makeThisPos(); return thisPos; }
protected Fields resolveRemainderFields( Set<Scope> incomingScopes, Fields argumentFields ) { Fields fields = resolveIncomingOperationArgumentFields( getFirst( incomingScopes ) ); if( fields.isUnknown() ) return fields; return fields.subtract( argumentFields ); }
public void setFields( Fields declared ) { if( declared == null ) throw new IllegalArgumentException( "declared fields must not be null" ); if( declared.isUnknown() || declared.isAll() ) return; // if operation declared ARGS, then the arguments are a selector and must be forced to declared declared = Fields.asDeclaration( declared ); this.tupleEntry = new TupleEntry( declared, Tuple.size( declared.size() ), true ); }
public Fields getSourceFields() { if( partitionFields == null || scheme.getSourceFields().isUnknown() ) return scheme.getSourceFields(); return Fields.merge( scheme.getSourceFields(), partitionFields ); }
public int[] getPos( Fields declarator, Fields selector ) { if( !declarator.isUnknown() && elements.size() != declarator.size() ) throw new TupleException( "field declaration: " + declarator.print() + ", does not match tuple: " + print() ); return declarator.getPos( selector, size() ); }
private void onFoundGroups( FunctionCall<Pair<Matcher, TupleEntry>> functionCall, Matcher matcher, TupleEntry output ) { int count = matcher.groupCount(); // if UNKNOWN then the returned number fields will be of variable size // subsequently we must clear the tuple, and add the found values if( functionCall.getDeclaredFields().isUnknown() ) addGroupsToTuple( matcher, output, count ); else setGroupsOnTuple( matcher, output, count ); // this overcomes an issue in the planner resolver where if REPLACE is declared, the declared // fields for the current operation are expected to match the argument fields functionCall.getOutputCollector().add( output.getTuple() ); }
/** * Method appendNew appends the given TupleEntry instance to this instance. * * @param entry of type TupleEntry * @return TupleEntry */ public TupleEntry appendNew( TupleEntry entry ) { Fields appendedFields = fields.append( entry.fields.isUnknown() ? Fields.size( entry.tuple.size() ) : entry.fields ); Tuple appendedTuple = tuple.append( entry.tuple ); return new TupleEntry( appendedFields, appendedTuple ); }
final int[] getPos( Fields fields, int tupleSize ) { // test for key, as we stuff a null value int[] pos = getPosCache().get( fields ); if( !isUnknown() && pos != null ) return pos; if( fields.isAll() ) return putReturn( fields, null ); // return null, not getPos() if( isAll() ) return putReturn( fields, fields.getPos() ); // don't cache unknown if( size() == 0 && isUnknown() ) return translatePos( fields, tupleSize ); pos = translatePos( fields, size() ); return putReturn( fields, pos ); }
void verifyArguments( Fields argumentSelector ) { if( argumentSelector.isUnknown() ) return; if( operation.getNumArgs() != Operation.ANY && argumentSelector.size() < operation.getNumArgs() ) throw new OperatorException( this, "resolved wrong number of arguments: " + argumentSelector.printVerbose() + ", expected: " + operation.getNumArgs() ); }
/** * Method isDeclarator returns true if this can be used as a declarator. Specifically if it is 'defined' or * {@link #UNKNOWN}, {@link #ALL}, {@link #ARGS}, {@link #GROUP}, or {@link #VALUES}. * * @return the declarator (type boolean) of this Fields object. */ public boolean isDeclarator() { return isUnknown() || isNone() || isAll() || isArguments() || isGroup() || isValues() || isDefined(); }
@Override public void prepare( FlowProcess flowProcess, OperationCall<Pair<Pattern, TupleEntry>> operationCall ) { length = operationCall.getDeclaredFields().isUnknown() ? -1 : operationCall.getDeclaredFields().size(); TupleEntry tupleEntry = new TupleEntry( operationCall.getDeclaredFields(), Tuple.size( Math.max( 1, length ) ) ); operationCall.setContext( new Pair<>( getPattern(), tupleEntry ) ); }
public SparseTupleComparator( Fields valuesFields, Fields sortFields, Comparator defaultComparator ) { if( defaultComparator == null ) defaultComparator = DEFAULT; int size = valuesFields != null && !valuesFields.isUnknown() ? valuesFields.size() : sortFields.size(); comparators = new Comparator[ size ]; posMap = new int[ size ]; Comparator[] sortFieldComparators = sortFields.getComparators(); // returns a copy for( int i = 0; i < sortFields.size(); i++ ) { Comparable field = sortFields.get( i ); int pos = valuesFields != null ? valuesFields.getPos( field ) : i; comparators[ i ] = sortFieldComparators[ i ]; posMap[ i ] = pos; if( comparators[ i ] == null ) comparators[ i ] = defaultComparator; } }
@Override public Fields retrieveSourceFields( FlowProcess<? extends Properties> process, Tap tap ) { if( !skipHeader || !getSourceFields().isUnknown() ) return getSourceFields(); // no need to open them all if( tap instanceof CompositeTap ) tap = (Tap) ( (CompositeTap) tap ).getChildTaps().next(); // can read either a single file, or search the directory for a file if( tap instanceof TapWith ) tap = ( (TapWith) tap ).withScheme( new TextLine( new Fields( "line" ), compressor, charsetName ) ).asTap(); else tap = new DirTap( new TextLine( new Fields( "line" ), compressor, charsetName ), tap.getIdentifier() ); setSourceFields( delimitedParser.parseFirstLine( process, tap ) ); return getSourceFields(); }
@Override public Fields retrieveSourceFields( FlowProcess<? extends Configuration> flowProcess, Tap tap ) { if( !skipHeader || !getSourceFields().isUnknown() ) return getSourceFields(); // no need to open them all if( tap instanceof CompositeTap ) tap = (Tap) ( (CompositeTap) tap ).getChildTaps().next(); // should revert to file:// (Lfs) if tap is Lfs if( tap instanceof TapWith ) tap = ( (TapWith) tap ).withScheme( new TextLine( new Fields( "line" ), charsetName ) ).asTap(); else tap = new Hfs( new TextLine( new Fields( "line" ), charsetName ), tap.getFullIdentifier( flowProcess ) ); setSourceFields( delimitedParser.parseFirstLine( flowProcess, tap ) ); return getSourceFields(); }
@Override public Fields retrieveSourceFields( FlowProcess<? extends Configuration> flowProcess, Tap tap ) { if( !skipHeader || !getSourceFields().isUnknown() ) return getSourceFields(); // no need to open them all if( tap instanceof CompositeTap ) tap = (Tap) ( (CompositeTap) tap ).getChildTaps().next(); // should revert to file:// (Lfs) if tap is Lfs if( tap instanceof TapWith ) tap = ( (TapWith) tap ).withScheme( new TextLine( new Fields( "line" ), charsetName ) ).asTap(); else tap = new Hfs( new TextLine( new Fields( "line" ), charsetName ), tap.getFullIdentifier( flowProcess ) ); setSourceFields( delimitedParser.parseFirstLine( flowProcess, tap ) ); return getSourceFields(); }