@Override public void register(DrillOperatorTable operatorTable) { for (String name : Sets.union(methodsGenericUDF.asMap().keySet(), methodsUDF.asMap().keySet())) { operatorTable.addOperatorWithoutInference(name, new HiveUDFOperatorWithoutInference(name.toUpperCase())); operatorTable.addOperatorWithInference(name, new HiveUDFOperator(name.toUpperCase(), new HiveSqlReturnTypeInference())); } }
for (Class<? extends GenericUDF> clazz: methodsGenericUDF.get(funcName)) { holder = matchAndCreateGenericUDFHolder(clazz, argTypes, argOIs); if (holder != null) { for (Class<? extends UDF> clazz : methodsUDF.get(funcName)) { holder = matchAndCreateUDFHolder(call.getName(), clazz, argTypes, argOIs); if (holder != null) {
private <I> void register(Class<? extends I> clazz, ArrayListMultimap<String, Class<? extends I>> methods) { Description desc = clazz.getAnnotation(Description.class); Stream<String> namesStream; if (desc != null) { namesStream = Stream.of(desc.name().split(",")) .map(String::trim); } else { namesStream = Stream.of(clazz) .map(Class::getName) .map(name -> name.replace('.', '_')); } // Checks specified array of function names whether they should be replaced // using FUNCTION_REPLACE_MAP map. namesStream.map(String::toLowerCase) .map(functionName -> FUNCTION_REPLACE_MAP.getOrDefault(functionName, functionName)) .forEach(name -> methods.put(name, clazz)); UDFType type = clazz.getAnnotation(UDFType.class); if (type != null && !type.deterministic()) { nonDeterministicUDFs.add(clazz); } }
@SuppressWarnings("resource") public void build(VectorContainer outputContainer) throws SchemaChangeException { outputContainer.clear(); if (batches.keySet().size() > 1) { throw new SchemaChangeException("Sort currently only supports a single schema."); if (batches.size() > Character.MAX_VALUE) { throw new SchemaChangeException("Sort cannot work on more than %d batches at a time.", (int) Character.MAX_VALUE); if (batches.keys().size() < 1) { assert false : "Invalid to have an empty set of batches with no schemas."; BatchSchema schema = batches.keySet().iterator().next(); List<RecordBatchData> data = batches.get(schema); ArrayListMultimap<MaterializedField, ValueVector> vectors = ArrayListMultimap.create(); for (RecordBatchData rbd : batches.values()) { for (ValueVector v : rbd.getVectors()) { vectors.put(v.getField(), v); List<ValueVector> v = vectors.get(f); outputContainer.addHyperList(v, false);
senderToReceiversMapping = ArrayListMultimap.create(); receiverToSenderMapping = Maps.newHashMap(); ArrayListMultimap<DrillbitEndpoint, Integer> endpointReceiverList = ArrayListMultimap.create(); endpointReceiverList.put(receiverLocation, receiverFragmentId); receiverFragmentId++; final List<Integer> receiverMinorFragmentIds = endpointReceiverList.get(senderLocation); senderToReceiversMapping.put(senderFragmentId, new MinorFragmentEndpoint(receiverId, receiverLocations.get(receiverId)));
public BroadcastSenderRootExec(RootFragmentContext context, RecordBatch incoming, BroadcastSender config) throws OutOfMemoryException { super(context, context.newOperatorContext(config, null), config); this.ok = true; this.incoming = incoming; this.config = config; this.handle = context.getHandle(); List<MinorFragmentEndpoint> destinations = config.getDestinations(); ArrayListMultimap<DrillbitEndpoint, Integer> dests = ArrayListMultimap.create(); for(MinorFragmentEndpoint destination : destinations) { dests.put(destination.getEndpoint(), destination.getId()); } int destCount = dests.keySet().size(); int i = 0; this.tunnels = new AccountingDataTunnel[destCount]; this.receivingMinorFragments = new int[destCount][]; for(final DrillbitEndpoint ep : dests.keySet()){ List<Integer> minorsList= dests.get(ep); int[] minorsArray = new int[minorsList.size()]; int x = 0; for(Integer m : minorsList){ minorsArray[x++] = m; } receivingMinorFragments[i] = minorsArray; tunnels[i] = context.getDataTunnel(ep); i++; } }
final Map<String, DrillSqlAggOperator.DrillSqlAggOperatorBuilder> mapAgg = Maps.newHashMap(); for (Entry<String, Collection<DrillFuncHolder>> function : registeredFunctions.entrySet()) { final ArrayListMultimap<Pair<Integer, Integer>, DrillFuncHolder> functions = ArrayListMultimap.create(); final ArrayListMultimap<Integer, DrillFuncHolder> aggregateFunctions = ArrayListMultimap.create(); final String name = function.getKey().toUpperCase(); boolean isDeterministic = true; final int paramCount = func.getParamCount(); if(func.isAggregating()) { aggregateFunctions.put(paramCount, func); } else { final Pair<Integer, Integer> argNumberRange; argNumberRange = Pair.of(func.getParamCount(), func.getParamCount()); functions.put(argNumberRange, func); for (Entry<Pair<Integer, Integer>, Collection<DrillFuncHolder>> entry : functions.asMap().entrySet()) { final Pair<Integer, Integer> range = entry.getKey(); final int max = range.getRight(); .setNiladic(isNiladic); for (Entry<Integer, Collection<DrillFuncHolder>> entry : aggregateFunctions.asMap().entrySet()) { if(!mapAgg.containsKey(name)) { mapAgg.put(name, new DrillSqlAggOperator.DrillSqlAggOperatorBuilder().setName(name));
/** * Add another record batch to the set of record batches. TODO: Refactor this and other {@link #add * (RecordBatchData)} method into one method. * @param batch * @return True if the requested add completed successfully. Returns false in the case that this builder is full and cannot receive additional packages. * @throws SchemaChangeException */ public boolean add(VectorAccessible batch) { if (batch.getSchema().getSelectionVectorMode() == SelectionVectorMode.FOUR_BYTE) { throw new UnsupportedOperationException("A sort cannot currently work against a sv4 batch."); } if (batch.getRecordCount() == 0 && batches.size() > 0) { return true; // skip over empty record batches. } long batchBytes = getSize(batch); if (batchBytes == 0 && batches.size() > 0) { return true; } if (runningBatches >= Character.MAX_VALUE) { return false; // allowed in batch. } if (!reservation.add(batch.getRecordCount() * 4)) { return false; // sv allocation available. } RecordBatchData bd = new RecordBatchData(batch, allocator); runningBatches++; batches.put(batch.getSchema(), bd); recordCount += bd.getRecordCount(); return true; }
public List<VectorContainer> getHeldRecordBatches() { ArrayList<VectorContainer> containerList = Lists.newArrayList(); for (BatchSchema bs : batches.keySet()) { for (RecordBatchData bd : batches.get(bs)) { VectorContainer c = bd.getContainer(); c.setRecordCount(bd.getRecordCount()); containerList.add(c); } } batches.clear(); return containerList; }
protected AbstractParquetGroupScan(AbstractParquetGroupScan that) { super(that); this.columns = that.columns == null ? null : new ArrayList<>(that.columns); this.parquetTableMetadata = that.parquetTableMetadata; this.rowGroupInfos = that.rowGroupInfos == null ? null : new ArrayList<>(that.rowGroupInfos); this.filter = that.filter; this.endpointAffinities = that.endpointAffinities == null ? null : new ArrayList<>(that.endpointAffinities); this.mappings = that.mappings == null ? null : ArrayListMultimap.create(that.mappings); this.parquetGroupScanStatistics = that.parquetGroupScanStatistics == null ? null : new ParquetGroupScanStatistics(that.parquetGroupScanStatistics); this.fileSet = that.fileSet == null ? null : new HashSet<>(that.fileSet); this.entries = that.entries == null ? null : new ArrayList<>(that.entries); this.readerConfig = that.readerConfig; this.matchAllRowGroups = that.matchAllRowGroups; }
/** * Creates a new, empty {@code ArrayListMultimap} with the default initial capacities. * * <p>This method will soon be deprecated in favor of {@code * MultimapBuilder.hashKeys().arrayListValues().build()}. */ public static <K, V> ArrayListMultimap<K, V> create() { return new ArrayListMultimap<K, V>(); }
public boolean isEmpty() { return batches.isEmpty(); }
/** * Reduces the memory used by this {@code ArrayListMultimap}, if feasible. * * @deprecated For a {@link ListMultimap} that automatically trims to size, use {@link * ImmutableListMultimap}. If you need a mutable collection, remove the {@code trimToSize} * call, or switch to a {@code HashMap<K, ArrayList<V>>}. This method is scheduled for removal * in April 2019. */ @Deprecated public void trimToSize() { for (Collection<V> collection : backingMap().values()) { ArrayList<V> arrayList = (ArrayList<V>) collection; arrayList.trimToSize(); } }
receiverToSenderMapping = ArrayListMultimap.create(); ArrayListMultimap<DrillbitEndpoint, Integer> endpointSenderList = ArrayListMultimap.create(); endpointSenderList.put(senderLocation, senderFragmentId); senderFragmentId++; List<Integer> senderFragmentIds = endpointSenderList.get(receiverLocation); receiverToSenderMapping.put(receiverFragmentId, new MinorFragmentEndpoint(senderId, senderLocations.get(senderId)));
@SuppressWarnings("resource") public void add(RecordBatchData rbd) { long batchBytes = getSize(rbd.getContainer()); if (batchBytes == 0 && batches.size() > 0) { return; } if (runningBatches >= Character.MAX_VALUE) { final String errMsg = String.format("Tried to add more than %d number of batches.", (int) Character.MAX_VALUE); logger.error(errMsg); throw new DrillRuntimeException(errMsg); } if (!reservation.add(rbd.getRecordCount() * 4)) { final String errMsg = String.format("Failed to pre-allocate memory for SV. " + "Existing recordCount*4 = %d, " + "incoming batch recordCount*4 = %d", recordCount * 4, rbd.getRecordCount() * 4); logger.error(errMsg); throw new DrillRuntimeException(errMsg); } if (rbd.getRecordCount() == 0 && batches.size() > 0) { rbd.getContainer().zeroVectors(); SelectionVector2 sv2 = rbd.getSv2(); if (sv2 != null) { sv2.clear(); } return; } runningBatches++; batches.put(rbd.getContainer().getSchema(), rbd); recordCount += rbd.getRecordCount(); }
/** * Returns list of functions with list of function signatures for each functions. * Uses guava {@link ListMultimap} structure to return data. * If no functions present, will return empty {@link ListMultimap}. * This is read operation, so several users can perform this operation at the same time. * * @return all functions which their signatures */ public ListMultimap<String, String> getAllFunctionsWithSignatures() { try (@SuppressWarnings("unused") Closeable lock = readLock.open()) { ListMultimap<String, String> functionsWithSignatures = ArrayListMultimap.create(); for (Map.Entry<String, Map<String, DrillFuncHolder>> function : functions.entrySet()) { functionsWithSignatures.putAll(function.getKey(), new ArrayList<>(function.getValue().keySet())); } return functionsWithSignatures; } }
/** * Constructs an {@code ArrayListMultimap} with the same mappings as the specified multimap. * * <p>This method will soon be deprecated in favor of {@code * MultimapBuilder.hashKeys().arrayListValues().build(multimap)}. * * @param multimap the multimap whose contents are copied to this multimap */ public static <K, V> ArrayListMultimap<K, V> create(Multimap<? extends K, ? extends V> multimap) { return new ArrayListMultimap<K, V>(multimap); }
/** * When the option planner.type_inference.enable is turned off, the operators which are added via this method * will be used. */ public void addOperatorWithoutInference(String name, SqlOperator op) { drillOperatorsWithoutInference.add(op); drillOperatorsWithoutInferenceMap.put(name.toLowerCase(), op); }