@Override public Long finalAggregate(Long aggregate, TupleEntry partialAggregate) { return aggregate += partialAggregate.getLong(0); }
/** * Method fieldToLong returns a {@link Function} that returns the long in the given named field or position. * * @param fields the field to select, only first field will be honored * @return the long value in the selected field */ public static ToLongFunction<TupleEntry> fieldToLong( Fields fields ) { Objects.requireNonNull( fields ); return value -> value.getLong( fields ); }
/** * Method fieldToLong returns a {@link Function} that returns the long in the given named field or position. * * @param name the field to select * @return the long value in the selected field */ public static ToLongFunction<TupleEntry> fieldToLong( Comparable name ) { Objects.requireNonNull( name ); return value -> value.getLong( name ); }
@Override public QDigest partialAggregate(QDigest aggregate, TupleEntry nextValue) { aggregate.offer(nextValue.getLong(0)); return aggregate; }
@Override public Long partialAggregate(Long aggregate, TupleEntry nextValue) { return update(aggregate, nextValue.getLong(0)); }
@Override public Long finalAggregate(Long aggregate, TupleEntry partialAggregate) { return update(aggregate, partialAggregate.getLong(0)); }
public void operate(FlowProcess process, FunctionCall call) { long val = call.getArguments().getLong(0); ByteBuffer buffer = ByteBuffer.allocate(8); buffer.putLong(val); call.getOutputCollector().add( new Tuple(buffer.array())); } }
aggregate[i] = 0L; aggregate[i] = ((Long)aggregate[i]) + nextValue.getLong(i); break; case DOUBLE:
@Override public boolean isRemove(FlowProcess flowProcess, FilterCall filterCall) { Long value = filterCall.getArguments().getLong(fieldName); if (counterEnum == null) { flowProcess.increment(counterGroup, counterName, value); } else { flowProcess.increment(counterEnum, value); } return false; } }
@Override public void aggregate( FlowProcess flowProcess, AggregatorCall<Context> aggregatorCall ) { Context context = aggregatorCall.getContext(); TupleEntry arguments = aggregatorCall.getArguments(); if( arguments.getObject( 0 ) == null ) { context.nulls++; return; } context.sum += arguments.getDouble( 0 ); context.count += arguments.getLong( 1 ); }
@Override public void operate( FlowProcess flowProcess, FunctionCall<Pair<SimpleDateFormat, TupleEntry>> functionCall ) { long ts = functionCall.getArguments().getLong( 0 ); Calendar calendar = getCalendar(); calendar.setTimeInMillis( ts ); String formatted = functionCall.getContext().getLhs().format( calendar.getTime() ); functionCall.getContext().getRhs().setString( 0, formatted ); functionCall.getOutputCollector().add( functionCall.getContext().getRhs() ); } }
@Test public void testCoerceCanonical() { final SimpleDateFormat dateFormat = new SimpleDateFormat( "dd/MMM/yyyy:HH:mm:ss:SSS Z" ); Date date = new Date(); String stringDate = dateFormat.format( date ); CoercibleType coercible = new DateType( "dd/MMM/yyyy:HH:mm:ss:SSS Z", TimeZone.getDefault() ); Fields fields = Fields.size( 2 ).applyTypes( coercible, long.class ); Tuple tuple = new Tuple( date.getTime(), date.getTime() ); TupleEntry results = new TupleEntry( fields, tuple ); assertEquals( date.getTime(), results.getObject( 0 ) ); assertEquals( date.getTime(), results.getLong( 0 ) ); assertEquals( stringDate, results.getString( 0 ) ); assertEquals( date.getTime(), results.getObject( 1 ) ); assertEquals( date.getTime(), results.getLong( 1 ) ); assertEquals( Long.toString( date.getTime() ), results.getString( 1 ) ); }
private static BloomFilter mergeBloomParts(String tapPath, long numBloomBits, long splitSize, int numBloomHashes, long numElems, HashFunctionFactory hashFactory) throws IOException { FixedSizeBitSet bitSet = new FixedSizeBitSet(numBloomBits); if (FileSystemHelper.getFS().exists(new Path(tapPath))) { Hfs tap = new Hfs(new SequenceFile(new Fields("split", "filter")), tapPath); TupleEntryIterator itr = tap.openForRead(CascadingUtil.get().getFlowProcess()); while (itr.hasNext()) { TupleEntry cur = itr.next(); long split = cur.getLong(0); FixedSizeBitSet curSet = new FixedSizeBitSet(splitSize, ((BytesWritable)cur.getObject(1)).getBytes()); for (long i = 0; i < curSet.numBits(); i++) { if (curSet.get(i)) { bitSet.set(split * splitSize + i); } } } itr.close(); } return new BloomFilter(numBloomBits, numBloomHashes, bitSet, numElems, hashFactory); }
private void writePrimitive(TupleEntry record, PrimitiveType field) { switch (field.getPrimitiveTypeName()) { case BINARY: recordConsumer.addBinary(Binary.fromString(record.getString(field.getName()))); break; case BOOLEAN: recordConsumer.addBoolean(record.getBoolean(field.getName())); break; case INT32: recordConsumer.addInteger(record.getInteger(field.getName())); break; case INT64: recordConsumer.addLong(record.getLong(field.getName())); break; case DOUBLE: recordConsumer.addDouble(record.getDouble(field.getName())); break; case FLOAT: recordConsumer.addFloat(record.getFloat(field.getName())); break; case FIXED_LEN_BYTE_ARRAY: throw new UnsupportedOperationException("Fixed len byte array type not implemented"); case INT96: throw new UnsupportedOperationException("Int96 type not implemented"); default: throw new UnsupportedOperationException(field.getName() + " type not implemented"); } } }
private void writePrimitive(TupleEntry record, PrimitiveType field) { switch (field.getPrimitiveTypeName()) { case BINARY: recordConsumer.addBinary(Binary.fromString(record.getString(field.getName()))); break; case BOOLEAN: recordConsumer.addBoolean(record.getBoolean(field.getName())); break; case INT32: recordConsumer.addInteger(record.getInteger(field.getName())); break; case INT64: recordConsumer.addLong(record.getLong(field.getName())); break; case DOUBLE: recordConsumer.addDouble(record.getDouble(field.getName())); break; case FLOAT: recordConsumer.addFloat(record.getFloat(field.getName())); break; case FIXED_LEN_BYTE_ARRAY: throw new UnsupportedOperationException("Fixed len byte array type not implemented"); case INT96: throw new UnsupportedOperationException("Int96 type not implemented"); default: throw new UnsupportedOperationException(field.getName() + " type not implemented"); } } }
private void writePrimitive(TupleEntry record, PrimitiveType field) { switch (field.getPrimitiveTypeName()) { case BINARY: recordConsumer.addBinary(Binary.fromString(record.getString(field.getName()))); break; case BOOLEAN: recordConsumer.addBoolean(record.getBoolean(field.getName())); break; case INT32: recordConsumer.addInteger(record.getInteger(field.getName())); break; case INT64: recordConsumer.addLong(record.getLong(field.getName())); break; case DOUBLE: recordConsumer.addDouble(record.getDouble(field.getName())); break; case FLOAT: recordConsumer.addFloat(record.getFloat(field.getName())); break; case FIXED_LEN_BYTE_ARRAY: throw new UnsupportedOperationException("Fixed len byte array type not implemented"); case INT96: throw new UnsupportedOperationException("Int96 type not implemented"); default: throw new UnsupportedOperationException(field.getName() + " type not implemented"); } } }
private void writePrimitive(TupleEntry record, PrimitiveType field) { switch (field.getPrimitiveTypeName()) { case BINARY: recordConsumer.addBinary(Binary.fromString(record.getString(field.getName()))); break; case BOOLEAN: recordConsumer.addBoolean(record.getBoolean(field.getName())); break; case INT32: recordConsumer.addInteger(record.getInteger(field.getName())); break; case INT64: recordConsumer.addLong(record.getLong(field.getName())); break; case DOUBLE: recordConsumer.addDouble(record.getDouble(field.getName())); break; case FLOAT: recordConsumer.addFloat(record.getFloat(field.getName())); break; case FIXED_LEN_BYTE_ARRAY: throw new UnsupportedOperationException("Fixed len byte array type not implemented"); case INT96: throw new UnsupportedOperationException("Int96 type not implemented"); default: throw new UnsupportedOperationException(field.getName() + " type not implemented"); } } }
assertEquals( date.getTime(), results.getLong( 0 ) ); assertEquals( stringDate, results.getString( 0 ) ); assertEquals( date.getTime(), results.getLong( 1 ) ); assertEquals( stringDate, results.getString( 1 ) ); assertEquals( date.getTime(), results.getLong( 2 ) ); assertEquals( stringDate, results.getString( 2 ) ); assertEquals( date.getTime(), results.getLong( 4 ) ); assertEquals( (int) date.getTime(), results.getInteger( 4 ) ); assertEquals( Long.toString( date.getTime() ), results.getString( 4 ) );
@Test public void testCoerceCanonicalUnknown() { Tuple tuple = new Tuple( 1 ); TupleEntry results = new TupleEntry( Fields.UNKNOWN, tuple ); assertEquals( 1, results.getObject( 0 ) ); assertEquals( 1, results.getInteger( 0 ) ); assertEquals( 1, results.getShort( 0 ) ); assertEquals( 1L, results.getLong( 0 ) ); assertEquals( 1.0F, results.getFloat( 0 ) ); assertEquals( 1.0D, results.getDouble( 0 ) ); assertEquals( "1", results.getString( 0 ) ); }