private static String createMessage( Fields sourceFields, Fields selectorFields ) { return "could not select fields: " + selectorFields.printVerbose() + ", from: " + sourceFields.printVerbose(); } }
private String printSafe( Fields fields ) { if( fields != null ) return fields.printVerbose(); return ""; }
private static String createMessage( Pipe pipe, Fields incomingFields, Fields declaredFields, Fields outputSelector ) { String message = "unable to resolve output selector: " + outputSelector.printVerbose() + ", with incoming: " + incomingFields.printVerbose() + " and declared: " + declaredFields.printVerbose(); return TraceUtil.formatTrace( pipe, message ); }
private static String createMessage( Pipe pipe, Kind kind, Fields incomingFields, Fields argumentSelector ) { String message = "unable to resolve " + kind + " selector: " + argumentSelector.printVerbose() + ", with incoming: " + incomingFields.printVerbose(); return TraceUtil.formatTrace( pipe, message ); }
private static String createMessage( Tap tap, Fields incomingFields, Fields selectorFields ) { String message = "unable to resolve scheme sink selector: " + selectorFields.printVerbose() + ", with incoming: " + incomingFields.printVerbose(); return TraceUtil.formatTrace( tap.getScheme(), message ); } }
private void verify() { if( fieldDeclaration.size() != 1 ) throw new IllegalArgumentException( "num fields in fieldDeclaration must be one, found: " + fieldDeclaration.printVerbose() ); }
public static void printOperationInternal( Operation operation, StringBuffer buffer, Scope scope ) { Class<? extends Operation> type = operation.getClass(); if( type.getSimpleName().length() != 0 ) buffer.append( type.getSimpleName() ); else buffer.append( type.getName() ); // should get something for an anonymous inner class if( scope.getOperationDeclaredFields() != null ) buffer.append( "[decl:" ).append( scope.getOperationDeclaredFields().printVerbose() ).append( "]" ); if( operation.getNumArgs() != ANY ) buffer.append( "[args:" ).append( operation.getNumArgs() ).append( "]" ); }
@Override public String toString() { StringBuilder buffer = new StringBuilder( super.toString() ); buffer.append( "[by:" ); for( String name : keyFieldsMap.keySet() ) { if( keyFieldsMap.size() > 1 ) buffer.append( " " ).append( name ).append( ":" ); buffer.append( keyFieldsMap.get( name ).printVerbose() ); } if( isSelfJoin() ) buffer.append( "[numSelfJoins:" ).append( numSelfJoins ).append( "]" ); buffer.append( "]" ); return buffer.toString(); }
public DelimitedPartition( Fields partitionFields, String delimiter, String postfix ) { if( partitionFields == null ) throw new IllegalArgumentException( "partitionFields must not be null" ); if( !partitionFields.isDefined() ) throw new IllegalArgumentException( "partitionFields must be defined, got: " + partitionFields.printVerbose() ); this.partitionFields = partitionFields; this.delimiter = delimiter == null ? PATH_DELIM : delimiter; postfix = Util.isEmpty( postfix ) ? null : postfix.startsWith( this.delimiter ) ? postfix.substring( this.delimiter.length() ) : postfix; this.numSplits = partitionFields.size() + ( postfix != null ? postfix.split( this.delimiter ).length : 0 ); this.postfix = postfix == null ? null : delimiter + postfix; // prefix the postfix w/ the delimiter }
public PartitionScheme( Scheme scheme, Fields partitionFields ) { this.scheme = scheme; if( partitionFields == null || partitionFields.isAll() ) this.partitionFields = null; else if( partitionFields.isDefined() ) this.partitionFields = partitionFields; else throw new IllegalArgumentException( "partitionFields must be defined or the ALL substitution, got: " + partitionFields.printVerbose() ); }
private Tuple selectTupleFrom( TupleEntry tupleEntry, Fields expectedFields ) { try { return tupleEntry.selectTuple( expectedFields ); } catch( TupleException exception ) { Fields givenFields = tupleEntry.getFields(); String string = "given TupleEntry fields: " + givenFields.printVerbose(); string += " do not match the operation declaredFields: " + expectedFields.printVerbose(); string += ", operations must emit tuples that match the fields they declare as output"; throw new TupleException( string, exception ); } }
@Override public void notifyWriteSpillBegin( Spillable spillable, int spillSize, String spillReason ) { int numFiles = spillable.spillCount(); if( numFiles % 10 == 0 ) { LOG.info( "spilling class: {}, spilling group: {}, on grouping: {}, num times: {}, with reason: {}", this.splillingClazz.getSimpleName(), new Object[]{joinField.printVerbose(), spillable.getGrouping().print(), numFiles + 1, spillReason} ); Runtime runtime = Runtime.getRuntime(); long freeMem = runtime.freeMemory() / 1024 / 1024; long maxMem = runtime.maxMemory() / 1024 / 1024; long totalMem = runtime.totalMemory() / 1024 / 1024; LOG.info( "spilling class: {}, mem on spill (mb), free: " + freeMem + ", total: " + totalMem + ", max: " + maxMem, this.splillingClazz.getSimpleName()); } LOG.info( "spilling class: {}, spilling {} tuples in list to file number {}", this.splillingClazz.getSimpleName(), spillSize, numFiles + 1 ); flowProcess.increment( Spill.Num_Spills_Written, 1 ); flowProcess.increment( Spill.Num_Tuples_Spilled, spillSize ); }
@Override public void notifyWriteSpillBegin( Spillable spillable, int spillSize, String spillReason ) { int numFiles = spillable.spillCount(); if( numFiles % 10 == 0 ) { LOG.info( "spilling group: {}, on grouping: {}, num times: {}, with reason: {}", new Object[]{joinField.printVerbose(), spillable.getGrouping().print(), numFiles + 1, spillReason} ); Runtime runtime = Runtime.getRuntime(); long freeMem = runtime.freeMemory() / 1024 / 1024; long maxMem = runtime.maxMemory() / 1024 / 1024; long totalMem = runtime.totalMemory() / 1024 / 1024; LOG.info( "mem on spill (mb), free: " + freeMem + ", total: " + totalMem + ", max: " + maxMem ); } LOG.info( "spilling {} tuples in list to file number {}", spillSize, numFiles + 1 ); flowProcess.increment( Spill.Num_Spills_Written, 1 ); flowProcess.increment( Spill.Num_Tuples_Spilled, spillSize ); }
@Override public void notifyWriteSpillBegin( Spillable spillable, int spillSize, String spillReason ) { int numFiles = spillable.spillCount(); if( numFiles % 10 == 0 ) { LOG.info( "spilling group: {}, on grouping: {}, num times: {}, with reason: {}", new Object[]{joinField.printVerbose(), spillable.getGrouping().print(), numFiles + 1, spillReason} ); Runtime runtime = Runtime.getRuntime(); long freeMem = runtime.freeMemory() / 1024 / 1024; long maxMem = runtime.maxMemory() / 1024 / 1024; long totalMem = runtime.totalMemory() / 1024 / 1024; LOG.info( "mem on spill (mb), free: " + freeMem + ", total: " + totalMem + ", max: " + maxMem ); } LOG.info( "spilling {} tuples in list to file number {}", spillSize, numFiles + 1 ); flowProcess.increment( Spill.Num_Spills_Written, 1 ); flowProcess.increment( Spill.Num_Tuples_Spilled, spillSize ); }
protected static Comparable asFieldName( Comparable fieldName ) { if( fieldName instanceof Fields ) { Fields fields = (Fields) fieldName; if( !fields.isDefined() ) throw new TupleException( "given Fields instance must explicitly declare one field name or position: " + fields.printVerbose() ); fieldName = fields.get( 0 ); } return fieldName; }
private String print( TupleEntry tupleEntry ) { if( tupleEntry == null || tupleEntry.getFields() == null ) return "[uninitialized]"; else if( tupleEntry.getTuple() == null ) return "fields: " + tupleEntry.getFields().printVerbose(); else return "fields: " + tupleEntry.getFields().printVerbose() + " tuple: " + tupleEntry.getTuple().print(); } }
void verifyDeclaredFields( Fields declared ) { if( declared.isDefined() && declared.size() == 0 ) throw new OperatorException( this, "field declaration: " + getFieldDeclaration().printVerbose() + ", resolves to an empty field set, current grouping is on all fields" ); }
void verifyOutputSelector( Fields outputSelector ) { if( outputSelector.isDefined() && outputSelector.size() == 0 ) throw new OperatorException( this, "output selector: " + getOutputSelector().printVerbose() + ", resolves to an empty field set, current grouping is on all fields" ); }
void verifyArguments( Fields argumentSelector ) { if( argumentSelector.isUnknown() ) return; if( operation.getNumArgs() != Operation.ANY && argumentSelector.size() < operation.getNumArgs() ) throw new OperatorException( this, "resolved wrong number of arguments: " + argumentSelector.printVerbose() + ", expected: " + operation.getNumArgs() ); }
protected void performTest( String inputData, Fields predictedFields, Fields expectedFields, EnsembleSpec<TreeSpec> ensembleSpec ) throws IOException { Pipe pipe = new Pipe( "head" ); pipe = new Discard( pipe, predictedFields ); pipe = new ParallelEnsembleAssembly( pipe, ensembleSpec ); pipe = new Pipe( "tail", pipe ); Tap source = getPlatform().getDelimitedFile( expectedFields.append( predictedFields ), true, ",", "\"", DATA_PATH + inputData, SinkMode.KEEP ); Tap sink = getPlatform().getDelimitedFile( Fields.ALL, true, ",", "\"", getResultPath(), SinkMode.REPLACE ); FlowDef flowDef = FlowDef.flowDef() .addSource( "head", source ) .addSink( "tail", sink ) .addTail( pipe ); Flow flow = getPlatform().getFlowConnector().connect( flowDef ); flow.writeDOT( getFlowPlanPath() + "/plan.dot" ); flow.complete(); Fields sourceSelector = source.getSourceFields(); Fields sinkSelector = sink.getSinkFields(); LOG.debug( "source select = {}", sourceSelector.printVerbose() ); LOG.debug( "sink select = {}", sinkSelector.printVerbose() ); List<Tuple> sourceTuples = asList( flow, source, sourceSelector ); List<Tuple> sinkTuples = asList( flow, sink, sinkSelector ); assertEquals( sourceTuples, sinkTuples, 0.000001d ); } }