/** * This method joins each value in the collection with the given delimiter. All results are appended to the * given {@link StringBuffer} instance. * * @param buffer * @param collection * @param delim */ public static void join( StringBuffer buffer, Collection collection, String delim ) { join( buffer, collection, delim, false ); }
@Override public String getID() { if( id == null ) id = Util.createUniqueID(); return id; }
protected void sleepForPollingInterval() { Util.safeSleep( pollingInterval ); }
cfg.set(TupleSerializationProps.SERIALIZATION_TOKENS, Util.join(",", Util.removeNulls(tokens, id + "=" + lmw))); LogFactory.getLog(EsTap.class).trace(String.format("Registered Cascading serialization token %s for %s", id, lmw)); return;
public static Object invokeStaticMethod( String typeString, String methodName, Object[] parameters, Class[] parameterTypes ) { Class type = loadClass( typeString ); return invokeStaticMethod( type, methodName, parameters, parameterTypes ); }
private void runComprehensiveCase( Boolean[] testCase, boolean useCollectionsComparator ) throws IOException { getPlatform().copyFromLocal( inputFileCrossNulls ); String test = Util.join( testCase, "_", true ) + "_" + useCollectionsComparator; String path = "comprehensive/" + test; Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileCrossNulls ); Tap sink = getPlatform().getDelimitedFile( new Fields( "num", "lower", "upper" ).applyTypes( Long.class, String.class, String.class ), " ", getOutputPath( path ), SinkMode.REPLACE ); sink.getScheme().setNumSinkParts( 1 ); Pipe pipe = new Pipe( "comprehensivesort" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ), "\\s" ) ); pipe = new Each( pipe, new Fields( "num" ), new Identity( Long.class ), Fields.REPLACE ); Fields groupFields = new Fields( "num" ); if( testCase[ 0 ] ) groupFields.setComparator( "num", useCollectionsComparator ? new NullSafeReverseComparator() : getPlatform().getLongComparator( true ) ); Fields sortFields = null; if( testCase[ 1 ] != null ) { sortFields = new Fields( "upper" ); if( testCase[ 1 ] ) sortFields.setComparator( "upper", useCollectionsComparator ? new NullSafeReverseComparator() : getPlatform().getStringComparator( true ) ); } pipe = new GroupBy( pipe, groupFields, sortFields, testCase[ 2 ] ); Map<Object, Object> properties = getProperties(); if( getPlatform().isMapReduce() && getPlatform().getNumMapTasks( properties ) != null ) getPlatform().setNumMapTasks( properties, 13 ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateCase( test, testCase, sink ); }
getPlatform().copyFromLocal( inputFileUpper ); Tap sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower ); Tap sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileUpper ); Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter ); pipeLower = new GroupBy( pipeLower, new Fields( "num" ) ); Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter ); pipeUpper = new GroupBy( pipeUpper, new Fields( "num" ) ); flow.start(); Util.safeSleep( 5000 );
Tap source = getPlatform().getDelimitedFile( new Fields( "number", "lower", "upper" ), " ", inputPath ); Flow firstFlow = getPlatform().getFlowConnector().connect( "first", source, sinkTap, new Pipe( "copy" ) ); Flow secondFlow = getPlatform().getFlowConnector().connect( "second", source, sinkTap, new Pipe( "copy" ) ); Util.safeSleep( 1000 ); // be safe, delay execution Flow thirdFlow = getPlatform().getFlowConnector().connect( "third", source, sinkTap, new Pipe( "copy" ) );
public DelimitedPartition( Fields partitionFields, String delimiter, String postfix ) { if( partitionFields == null ) throw new IllegalArgumentException( "partitionFields must not be null" ); if( !partitionFields.isDefined() ) throw new IllegalArgumentException( "partitionFields must be defined, got: " + partitionFields.printVerbose() ); this.partitionFields = partitionFields; this.delimiter = delimiter == null ? PATH_DELIM : delimiter; postfix = Util.isEmpty( postfix ) ? null : postfix.startsWith( this.delimiter ) ? postfix.substring( this.delimiter.length() ) : postfix; this.numSplits = partitionFields.size() + ( postfix != null ? postfix.split( this.delimiter ).length : 0 ); this.postfix = postfix == null ? null : delimiter + postfix; // prefix the postfix w/ the delimiter }
private static String emptyOrValue( Object value ) { if( value == null ) return ""; if( Util.isEmpty( value.toString() ) ) return ""; return value.toString(); }
String tailName = tail.getName(); String tailName = pipe.getName(); throw new PlannerException( "not all tail pipes bound to sink taps, remaining tail pipe names: [" + join( quote( tailNames, "'" ), ", " ) + "], remaining sink tap names: [" + join( quote( remainingSinks, "'" ), ", " ) + "]" ); remainingSinks.removeAll( asList( Pipe.names( flowTails ) ) ); throw new PlannerException( "not all sink taps bound to tail pipes, remaining sink tap names: [" + join( quote( remainingSinks, "'" ), ", " ) + "]" ); throw new PlannerException( "not all head pipes bound to source taps, remaining head pipe names: [" + join( quote( headNames, "'" ), ", " ) + "], remaining source tap names: [" + join( quote( remainingSources, "'" ), ", " ) + "]" ); throw new PlannerException( "not all source taps bound to head pipes, remaining source tap names: [" + join( quote( remainingSources, "'" ), ", " ) + "], remaining head pipe names: [" + join( quote( headNames, "'" ), ", " ) + "]" );
public static Map<String, Pipe> buildOpPipes( String prefix, Pipe pipe, AssemblyFactory assemblyFactory, Fields[] args_fields, Fields[] decl_fields, Fields[] select_fields, String functionValue, String runOnly ) { Map<String, Pipe> pipes = new LinkedHashMap<String, Pipe>(); for( int arg = 0; arg < args_fields.length; arg++ ) { Fields argFields = args_fields[ arg ]; for( int decl = 0; decl < decl_fields.length; decl++ ) { Fields declFields = decl_fields[ decl ]; for( int select = 0; select < select_fields.length; select++ ) { Fields selectFields = select_fields[ select ]; String name; if( prefix != null ) name = prefix + "." + Util.join( Fields.fields( argFields, declFields, selectFields ), "_" ); else name = Util.join( Fields.fields( argFields, declFields, selectFields ), "_" ); if( runOnly != null && !runOnly.equalsIgnoreCase( name ) ) continue; pipes.put( name, assemblyFactory.createAssembly( pipe, argFields, declFields, functionValue, selectFields ) ); } } } return pipes; }
public Integer getFirstOrdinal() { if( getOrdinals().isEmpty() ) return null; return Util.getFirst( getOrdinals() ); }
@Override public String toString() { String string; if( isOrdered() ) string = orderedToString(); else string = unorderedToString(); if( types != null ) string += " | " + Util.join( Util.simpleTypeNames( types ), ", " ); return string; }
private Tap decorateTap( Pipe pipe, Tap tempTap, String decoratorClassProp, String defaultDecoratorClassName ) { String decoratorClassName = PropertyUtil.getProperty( defaultProperties, pipe, decoratorClassProp ); if( Util.isEmpty( decoratorClassName ) ) decoratorClassName = defaultDecoratorClassName; if( Util.isEmpty( decoratorClassName ) ) return tempTap; LOG.info( "found decorator property: {}, with value: {}, wrapping tap: {}", decoratorClassProp, decoratorClassName, tempTap ); tempTap = Util.newInstance( decoratorClassName, tempTap ); return tempTap; }
private void verifyPipelines() { if( pipelineGraphs == null || pipelineGraphs.isEmpty() ) return; Set<FlowElement> allElements = createIdentitySet( nodeSubGraph.vertexSet() ); for( ElementGraph pipelineGraph : pipelineGraphs ) allElements.removeAll( pipelineGraph.vertexSet() ); if( !allElements.isEmpty() ) throw new IllegalStateException( "union of pipeline graphs for flow node are missing elements: " + Util.join( allElements, ", " ) ); }
protected static Job getJob( RunningJob runningJob ) { if( runningJob == null ) // if null, job hasn't been submitted return null; Job job = Util.returnInstanceFieldIfExistsSafe( runningJob, "job" ); if( job == null ) { LOG.warn( "unable to get underlying org.apache.hadoop.mapreduce.Job from org.apache.hadoop.mapred.RunningJob, task level task counters will be unavailable" ); return null; } return job; }
public static String setLog4jLevel( String logger, String level ) { // removing logj4 dependency // org.apache.log4j.Logger.getLogger( logger[ 0 ] ).setLevel( org.apache.log4j.Level.toLevel( logger[ 1 ] ) ); Object loggerObject = Util.invokeStaticMethod( "org.apache.log4j.Logger", "getLogger", new Object[]{logger}, new Class[]{String.class} ); Object levelObject = null; if( level != null ) levelObject = Util.invokeStaticMethod( "org.apache.log4j.Level", "toLevel", new Object[]{level}, new Class[]{String.class} ); Object oldLevel = Util.invokeInstanceMethod( loggerObject, "getLevel", new Object[]{}, new Class[]{} ); Util.invokeInstanceMethod( loggerObject, "setLevel", new Object[]{levelObject}, new Class[]{Util.loadClass( "org.apache.log4j.Level" )} ); if( oldLevel == null ) return null; return oldLevel.toString(); }