@Override public void start( FlowProcess flowProcess, AggregatorCall<Pair<Long[], Tuple>> aggregatorCall ) { aggregatorCall.getContext().getLhs()[ 0 ] = 0L; }
@Override public void aggregate( FlowProcess flowProcess, AggregatorCall<Pair<Long[], Tuple>> aggregatorCall ) { aggregatorCall.getContext().getLhs()[ 0 ] += 1L; }
n1 = next.getLhs(); n2 = next.getRhs();
public static ElementSubGraph asSubGraph( ElementGraph elementGraph, ElementGraph contractedGraph, Set<FlowElement> excludes ) { elementGraph = asExtentMaskedSubGraph( elementGraph ); // returns same instance if not bounded Pair<Set<FlowElement>, Set<Scope>> pair = findClosureViaFloydWarshall( directed( elementGraph ), directed( contractedGraph ), excludes ); Set<FlowElement> vertices = pair.getLhs(); Set<Scope> excludeEdges = pair.getRhs(); Set<Scope> scopes = new HashSet<>( elementGraph.edgeSet() ); scopes.removeAll( excludeEdges ); return new ElementSubGraph( elementGraph, vertices, scopes ); }
@Override public void start( FlowProcess flowProcess, AggregatorCall<Pair<Double[], Tuple>> aggregatorCall ) { aggregatorCall.getContext().getLhs()[ 0 ] = null; aggregatorCall.getContext().getRhs().set( 0, null ); }
@Override public void aggregate( FlowProcess flowProcess, AggregatorCall<Pair<Double[], Tuple>> aggregatorCall ) { TupleEntry arguments = aggregatorCall.getArguments(); if( arguments.getObject( 0 ) == null ) return; Double[] sum = aggregatorCall.getContext().getLhs(); double value = sum[ 0 ] == null ? 0 : sum[ 0 ]; sum[ 0 ] = value + arguments.getDouble( 0 ); }
public static BloomFilter createAndConsumeFilter(String bloomPartsDir, int maxHashes, int minHashes, long bloomFilterBits, int numSplits, double hllError, String bloomKeyCountsDir, HashFunctionFactory hashFactory, FilterConsumer filterConsumer) throws IOException, CardinalityMergeException { LOG.info("Bloom filter parts located in: " + bloomPartsDir); // This is the side bucket that the HyperLogLog writes to LOG.info("Getting key counts from: " + bloomKeyCountsDir); long prevJobTuples = getApproxDistinctKeysCount(hllError, bloomKeyCountsDir); Pair<Double, Integer> optimal = getOptimalFalsePositiveRateAndNumHashes(bloomFilterBits, prevJobTuples, minHashes, maxHashes); LOG.info("Counted about " + prevJobTuples + " distinct keys"); LOG.info("Using " + bloomFilterBits + " bits in the bloom filter"); LOG.info("Found a false positive rate of: " + optimal.getLhs()); LOG.info("Will use " + optimal.getRhs() + " bloom hashes"); long splitSize = getSplitSize(bloomFilterBits, numSplits); int numBloomHashes = optimal.getRhs(); BloomFilter filter; synchronized (BF_LOAD_LOCK) { // Load bloom filter parts and merge them. String path = bloomPartsDir + "/" + numBloomHashes; filter = mergeBloomParts(path, bloomFilterBits, splitSize, numBloomHashes, prevJobTuples, hashFactory); filterConsumer.accept(filter); } return filter; }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<SimpleDateFormat, TupleEntry>> functionCall ) { long ts = functionCall.getArguments().getLong( 0 ); Calendar calendar = getCalendar(); calendar.setTimeInMillis( ts ); String formatted = functionCall.getContext().getLhs().format( calendar.getTime() ); functionCall.getContext().getRhs().setString( 0, formatted ); functionCall.getOutputCollector().add( functionCall.getContext().getRhs() ); } }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<Matcher, TupleEntry>> functionCall ) { // coerce to string String value = functionCall.getArguments().getString( 0 ); // make safe if( value == null ) value = ""; TupleEntry output = functionCall.getContext().getRhs(); Matcher matcher = functionCall.getContext().getLhs().reset( value ); if( replaceAll ) output.setString( 0, matcher.replaceAll( replacement ) ); else output.setString( 0, matcher.replaceFirst( replacement ) ); functionCall.getOutputCollector().add( output ); }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<Matcher, TupleEntry>> functionCall ) { String value = functionCall.getArguments().getString( 0 ); if( value == null ) value = ""; Matcher matcher = functionCall.getContext().getLhs().reset( value ); while( matcher.find() ) { functionCall.getContext().getRhs().setString( 0, matcher.group() ); functionCall.getOutputCollector().add( functionCall.getContext().getRhs() ); } } }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<Matcher, TupleEntry>> functionCall ) { String value = functionCall.getArguments().getString( 0 ); if( value == null ) value = ""; Matcher matcher = functionCall.getContext().getLhs().reset( value ); if( !matcher.find() ) throw new OperationException( "could not match pattern: [" + getPatternString() + "] with value: [" + value + "]" ); TupleEntry output = functionCall.getContext().getRhs(); if( groups != null ) onGivenGroups( functionCall, matcher, output ); else onFoundGroups( functionCall, matcher, output ); }
@Override public void operate( FlowProcess flowProcess, FunctionCall<NestedBaseFunction.Context> functionCall ) { Node node = getNode( functionCall.getArguments() ); Set<Map.Entry<Fields, Pair<Predicate<?>, Pointer<Node>>>> entries = functionCall.getContext().pointers.entrySet(); for( Map.Entry<Fields, Pair<Predicate<?>, Pointer<Node>>> entry : entries ) { Fields key = entry.getKey(); Predicate<Object> predicate = (Predicate<Object>) entry.getValue().getLhs(); Pointer<Node> pointer = entry.getValue().getRhs(); Object argumentValue = functionCall.getArguments().getObject( key ); if( predicate.test( argumentValue ) ) { Node result = getLiteralNode( argumentValue ); pointer.set( node, result ); } } functionCall.getContext().result.set( 0, node ); functionCall.getOutputCollector().add( functionCall.getContext().result ); }
protected Tuple getResult( AggregatorCall<Pair<Long[], Tuple>> aggregatorCall ) { aggregatorCall.getContext().getRhs().set( 0, aggregatorCall.getContext().getLhs()[ 0 ] ); return aggregatorCall.getContext().getRhs(); } }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<Pattern, TupleEntry>> functionCall ) { String value = functionCall.getArguments().getString( 0 ); if( value == null ) value = ""; String[] split = functionCall.getContext().getLhs().split( value ); for( String string : split ) { TupleEntry tupleEntry = functionCall.getContext().getRhs(); tupleEntry.setString( 0, string ); functionCall.getOutputCollector().add( tupleEntry ); } } }
protected Tuple getResult( AggregatorCall<Pair<Double[], Tuple>> aggregatorCall ) { aggregatorCall.getContext().getRhs().set( 0, canonical.canonical( aggregatorCall.getContext().getLhs()[ 0 ] ) ); return aggregatorCall.getContext().getRhs(); }
private void assertGraphs( SimpleDirectedGraph<String, Object> full, SimpleDirectedGraph<String, Object> contracted, SimpleDirectedGraph<String, Object> result ) { // Set<String> vertices = ElementGraphs.findClosureViaBiConnected( full, contracted ); Set<String> vertices = ElementGraphs.findClosureViaFloydWarshall( full, contracted ).getLhs(); // Set<String> vertices = ElementGraphs.findClosureViaKShortest( full, contracted ); DirectedSubgraph<String, Object> subgraph = new DirectedSubgraph<>( full, vertices, null ); // System.out.println( "subgraph = " + subgraph ); SimpleDirectedGraph<String, Object> clone = new SimpleDirectedGraph<>( Object.class ); Graphs.addGraph( clone, subgraph ); assertEquals( result, clone ); }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<Pattern, TupleEntry>> functionCall ) { String value = functionCall.getArguments().getString( 0 ); if( value == null ) value = ""; TupleEntry output = functionCall.getContext().getRhs(); String[] split = functionCall.getContext().getLhs().split( value, length ); if( length == -1 ) { output.getTuple().clear(); for( String element : split ) output.getTuple().add( element ); } else { output.setCanonicalValues( split ); } functionCall.getOutputCollector().add( output ); } }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<SimpleDateFormat, TupleEntry>> functionCall ) { TupleEntry output = functionCall.getContext().getRhs(); try { String value = functionCall.getArguments().getString( 0 ); if( value == null ) // if null, return null for the field { output.setObject( 0, null ); // safe to call set, tuple is size of 1 functionCall.getOutputCollector().add( output ); return; } Date date = functionCall.getContext().getLhs().parse( value ); if( calendarFields == null ) output.setLong( 0, date.getTime() ); // safe to call set, tuple is size of 1 else makeCalendarFields( output, date ); } catch( ParseException exception ) { throw new OperationException( "unable to parse input value: " + functionCall.getArguments().getObject( 0 ), exception ); } functionCall.getOutputCollector().add( output ); }