public MapCombinerAggStateUpdater(CombinerAggregator agg, Fields groupFields, Fields inputFields) { _agg = agg; _groupFields = groupFields; _inputFields = inputFields; if (inputFields.size() != 1) { throw new IllegalArgumentException("Combiner aggs only take a single field as input. Got this instead: " + inputFields.toString()); } _factory = new ComboList.Factory(groupFields.size(), inputFields.size()); }
public List<Object> select(Fields selector, List<Object> tuple) { List<Object> ret = new ArrayList<>(selector.size()); for (String s : selector) { ret.add(tuple.get(_index.get(s))); } return ret; }
public static ValuePointer[] buildIndex(Fields fieldsOrder, Map<String, ValuePointer> pointers) { if(fieldsOrder.size()!=pointers.size()) { throw new IllegalArgumentException("Fields order must be same length as pointers map"); } ValuePointer[] ret = new ValuePointer[pointers.size()]; for(int i=0; i<fieldsOrder.size(); i++) { ret[i] = pointers.get(fieldsOrder.get(i)); } return ret; }
protected int getHashCode(List<Object> tuple) { if (groupFieldIndex.size() == 1) { return tuple.get(groupFieldIndex.get(0)).hashCode(); } else { List<Object> groupFieldValues = new ArrayList<>(groupFields.size()); for (Integer index : groupFieldIndex) { groupFieldValues.add(tuple.get(index)); } return groupFieldValues.hashCode(); } } }
public List<Object> select(Fields selector, List<Object> tuple) { List<Object> ret = new ArrayList<Object>(selector.size()); for(String s: selector) { ret.add(tuple.get(_index.get(s))); } return ret; }
@Override public void prepare(Map conf, TridentMultiReducerContext context) { int[] sizes = new int[_sideFields.size() + 1]; sizes[0] = _numGroupFields; for (int i = 0; i < _sideFields.size(); i++) { sizes[i + 1] = _sideFields.get(i).size(); } _factory = new ComboList.Factory(sizes); }
public GroupedAggregator(Aggregator agg, Fields group, Fields input, int outSize) { _groupFields = group; _inFields = input; _agg = agg; int[] sizes = new int[2]; sizes[0] = _groupFields.size(); sizes[1] = outSize; _fact = new ComboList.Factory(sizes); }
public MapReducerAggStateUpdater(ReducerAggregator agg, Fields groupFields, Fields inputFields) { _agg = agg; _groupFields = groupFields; _inputFields = inputFields; _factory = new ComboList.Factory(groupFields.size(), 1); }
@Override public void complete(JoinState state, TridentTuple group, TridentCollector collector) { List<List>[] sides = state.sides; boolean wasEmpty = state.numSidesReceived < sides.length; for (int i = 0; i < sides.length; i++) { if (sides[i].isEmpty() && _types.get(i) == JoinType.OUTER) { state.numSidesReceived++; sides[i].add(makeNullList(_sideFields.get(i).size())); } } if (wasEmpty && state.numSidesReceived == sides.length) { emitCrossJoin(state, collector, -1, null); } }
@Override public void emitBatch(long batchId, TridentCollector collector) { List<List<Object>> values = null; if (batches.containsKey(batchId)) { values = batches.get(batchId); } else { values = new ArrayList<>(); for (int i = 0; i < batchSize; i++) { List<Object> numbers = new ArrayList<>(); for (int x = 0; x < fields.size(); x++) { numbers.add(ThreadLocalRandom.current().nextInt(0, maxNumber + 1)); } values.add(numbers); } batches.put(batchId, values); } for (List<Object> value : values) { collector.emit(value); } }
public Tuple(TopologyContext context, List<Object> values, int taskId, String streamId, MessageId id) { super(); this.values = values; this.taskId = taskId; this.streamId = streamId; this.id = id; this.context = context; String componentId = context.getComponentId(taskId); Fields schema = context.getComponentOutputFields(componentId, streamId); if(values.size()!=schema.size()) { throw new IllegalArgumentException( "Tuple created with wrong number of fields. " + "Expected " + schema.size() + " fields but got " + values.size() + " fields"); } }
public FreshOutputFactory(Fields selfFields) { _fieldIndex = new HashMap<>(); for(int i=0; i<selfFields.size(); i++) { String field = selfFields.get(i); _fieldIndex.put(field, new ValuePointer(0, i, field)); } _index = ValuePointer.buildIndex(selfFields, _fieldIndex); }
public RootFactory(Fields inputFields) { index = new ValuePointer[inputFields.size()]; int i=0; for(String f: inputFields) { index[i] = new ValuePointer(0, i, f); i++; } fieldIndex = ValuePointer.buildFieldIndex(index); }
private IKvState<K, V> getSpecifiedKeyRangeState(Tuple input, Fields fields) { Object key = null; if (fields.size() == 1) { key = input.getValueByField(fields.get(0)); } else { List<Object> fieldedValuesTobeHash = Lists.newArrayList(); for (String field : fields) { fieldedValuesTobeHash.add(input.getValueByField(field)); } key = fieldedValuesTobeHash; } return keyRangeState.getRangeStateByKey(key); } }
public OperationOutputFactory(Factory parent, Fields selfFields) { _parent = parent; _fieldIndex = new HashMap<>(parent.getFieldIndex()); int myIndex = parent.numDelegates(); for(int i=0; i<selfFields.size(); i++) { String field = selfFields.get(i); _fieldIndex.put(field, new ValuePointer(myIndex, i, field)); } List<String> myOrder = new ArrayList<>(parent.getOutputFields()); Set<String> parentFieldsSet = new HashSet<>(myOrder); for(String f: selfFields) { if(parentFieldsSet.contains(f)) { throw new IllegalArgumentException( "Additive operations cannot add fields with same name as already exists. " + "Tried adding " + selfFields + " to " + parent.getOutputFields()); } myOrder.add(f); } _index = ValuePointer.buildIndex(new Fields(myOrder), _fieldIndex); }
if (_fieldLocations.size() != _outFields.size()) { throw new RuntimeException("Cannot find all outfields among sources");
public Stream join(List<Stream> streams, List<Fields> joinFields, Fields outFields, List<JoinType> mixed) { return multiReduce(strippedInputFields(streams, joinFields), groupedStreams(streams, joinFields), new JoinerMultiReducer(mixed, joinFields.get(0).size(), strippedInputFields(streams, joinFields)), outFields); }
outSizes[i] = outfields.size(); allOutFields.addAll(outfields.toList()); allInFields.addAll(infields.toList());
@Override public IAggregatableStream partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields) { Aggregator groupedAgg = new GroupedAggregator(agg, _groupFields, inputFields, functionFields.size()); Fields allInFields = TridentUtils.fieldsUnion(_groupFields, inputFields); Fields allOutFields = TridentUtils.fieldsConcat(_groupFields, functionFields); Stream s = _stream.partitionAggregate(allInFields, groupedAgg, allOutFields); return new GroupedStream(s, _groupFields); }
@Override public void prepare(Map conf, TridentMultiReducerContext context) { int[] sizes = new int[_sideFields.size() + 1]; sizes[0] = _numGroupFields; for(int i=0; i<_sideFields.size(); i++) { sizes[i+1] = _sideFields.get(i).size(); } _factory = new ComboList.Factory(sizes); }