Refine search
/** * Group the input pairs by the key of each pair. * @param listInput the list of pairs to group * @param <K> the key type * @param <V> the value type * @return a map representing a grouping of the * keys by the given input key type and list of values * in the grouping. */ public static <K,V> Map<K,List<V>> groupByKey(List<Pair<K,V>> listInput) { Map<K,List<V>> ret = new HashMap<>(); for(Pair<K,V> pair : listInput) { List<V> currList = ret.get(pair.getFirst()); if(currList == null) { currList = new ArrayList<>(); ret.put(pair.getFirst(),currList); } currList.add(pair.getSecond()); } return ret; }
/** * Merge the specified 2d arrays and masks. See {@link #mergeFeatures(INDArray[], INDArray[])} * and {@link #mergeLabels(INDArray[], INDArray[])} * * @param arrays Arrays to merge * @param masks Mask arrays to merge * @param inOutIdx Index to extract out before merging * @return Merged arrays and mask */ public static Pair<INDArray, INDArray> merge2d(@NonNull INDArray[][] arrays, INDArray[][] masks, int inOutIdx) { Pair<INDArray[], INDArray[]> p = selectColumnFromMDSData(arrays, masks, inOutIdx); return merge2d(p.getFirst(), p.getSecond()); }
/** * Extract out the specified column, and merge the specified label and label mask arrays * (i.e., concatenate the examples) * * @param labelsToMerge Features to merge. Will use featuresToMerge[all][inOutIdx] * @param labelMasksToMerge Mask arrays to merge. May be null * @return Merged features and mask. Mask may be null */ public static Pair<INDArray, INDArray> mergeLabels(@NonNull INDArray[][] labelsToMerge, INDArray[][] labelMasksToMerge, int inOutIdx) { Pair<INDArray[], INDArray[]> p = selectColumnFromMDSData(labelsToMerge, labelMasksToMerge, inOutIdx); return mergeLabels(p.getFirst(), p.getSecond()); }
/** * Extract out the specified column, and merge the specified features and mask arrays (i.e., concatenate the examples) * * @param featuresToMerge Features to merge. Will use featuresToMerge[all][inOutIdx] * @param featureMasksToMerge Mask arrays to merge. May be null * @return Merged features and mask. Mask may be null */ public static Pair<INDArray, INDArray> mergeFeatures(INDArray[][] featuresToMerge, INDArray[][] featureMasksToMerge, int inOutIdx) { Pair<INDArray[], INDArray[]> p = selectColumnFromMDSData(featuresToMerge, featureMasksToMerge, inOutIdx); return mergeFeatures(p.getFirst(), p.getSecond()); }
/** * Returns a {@link Set} view of the mappings contained in this map. * The applyTransformToDestination is backed by the map, so changes to the map are * reflected in the applyTransformToDestination, and vice-versa. If the map is modified * while an iteration over the applyTransformToDestination is in progress (except through * the iterator's own <tt>remove</tt> operation, or through the * <tt>setValue</tt> operation on a map entry returned by the * iterator) the results of the iteration are undefined. The applyTransformToDestination * supports element removal, which removes the corresponding * mapping from the map, via the <tt>Iterator.remove</tt>, * <tt>Set.remove</tt>, <tt>removeAll</tt>, <tt>retainAll</tt> and * <tt>clear</tt> operations. It does not support the * <tt>add</tt> or <tt>addAll</tt> operations. * * @return a applyTransformToDestination view of the mappings contained in this map */ public Set<Entry<K, T, V>> entrySet() { Set<Entry<K, T, V>> ret = new HashSet<>(); for (Pair<K, T> pair : backedMap.keySet()) { ret.add(new Entry<>(pair.getFirst(), pair.getSecond(), backedMap.get(pair))); } return ret; }
/** * Merge the specified time series (3d) arrays and masks. See {@link #mergeFeatures(INDArray[], INDArray[])} * and {@link #mergeLabels(INDArray[], INDArray[])} * * @param arrays Arrays to merge * @param masks Mask arrays to merge * @param inOutIdx Index to extract out before merging * @return Merged arrays and mask */ public static Pair<INDArray, INDArray> mergeTimeSeries(INDArray[][] arrays, INDArray[][] masks, int inOutIdx) { Pair<INDArray[], INDArray[]> p = selectColumnFromMDSData(arrays, masks, inOutIdx); return mergeTimeSeries(p.getFirst(), p.getSecond()); }
/** * Merge the specified 4d arrays and masks. See {@link #mergeFeatures(INDArray[], INDArray[])} * and {@link #mergeLabels(INDArray[], INDArray[])} * * @param arrays Arrays to merge * @param masks Mask arrays to merge * @param inOutIdx Index to extract out before merging * @return Merged arrays and mask */ public static Pair<INDArray, INDArray> merge4d(INDArray[][] arrays, INDArray[][] masks, int inOutIdx) { Pair<INDArray[], INDArray[]> p = selectColumnFromMDSData(arrays, masks, inOutIdx); return merge4d(p.getFirst(), p.getSecond()); }
/** * This method removes all elements except of top N by counter values * @param N */ public void keepTopNElements(int N){ PriorityQueue<Pair<T, Double>> queue = asPriorityQueue(); clear(); for (int e = 0; e < N; e++) { Pair<T, Double> pair = queue.poll(); if (pair != null) incrementCount(pair.getFirst(), pair.getSecond()); } }
/** * Merge all of the features arrays into one minibatch. * * @param featuresToMerge features to merge. Note that first index is the input array (example) index, the second * index is the input array. * Thus to merge 10 examples with 3 input arrays each, featuresToMerge will be indexed * like featuresToMerge[0..9][0..2] * @param featureMasksToMerge May be null. If non-null: feature masks to merge * @return Merged features, and feature masks. Note that feature masks may be added automatically, if required - even * if no feature masks were present originally */ public static Pair<INDArray[], INDArray[]> mergeFeatures(@NonNull INDArray[][] featuresToMerge, INDArray[][] featureMasksToMerge) { int nInArrs = featuresToMerge[0].length; INDArray[] outF = new INDArray[nInArrs]; INDArray[] outM = null; for (int i = 0; i < nInArrs; i++) { Pair<INDArray, INDArray> p = mergeFeatures(featuresToMerge, featureMasksToMerge, i); outF[i] = p.getFirst(); if (p.getSecond() != null) { if (outM == null) { outM = new INDArray[nInArrs]; } outM[i] = p.getSecond(); } } return new Pair<>(outF, outM); }
for(currMinibatchSize = 0; this.documentProvider.hasNext(); ++currMinibatchSize) { Pair<String, double[]> p = this.documentProvider.nextSentence(); List<String> tokens = this.tokenizeSentence(p.getFirst()); maxLength = Math.max(maxLength, tokens.size()); tokenizedSentences.add(new Pair(tokens, p.getSecond())); List<String> currSentence = tokenizedSentences.get(i).getFirst();
for(currMinibatchSize = 0; currMinibatchSize < num && this.documentProvider.hasNext(); ++currMinibatchSize) { Pair<String, double[]> p = this.documentProvider.nextSentence(); List<String> tokens = this.tokenizeSentence(p.getFirst()); maxLength = Math.max(maxLength, tokens.size()); tokenizedSentences.add(new Pair(tokens, p.getSecond())); double[] labelArray = tokenizedSentences.get(i).getSecond(); labelsData[i] = labelArray; List<String> currSentence = tokenizedSentences.get(i).getFirst(); sentenceLength = ((List)((Pair)tokenizedSentences.get(i)).getFirst()).size(); if(sentenceLength >= maxLength) { featuresMask.getRow(i).assign(Double.valueOf(1.0D));
if (!reverseMap.containsKey(pair.getFirst())) { if (pair.getFirst().contains("NextIteration")) { forwardMap.put(pair.getFirst(), fwdNodeId); reverseMap.put(pair.getFirst(), fwdNodeId); } else { throw new ND4JIllegalStateException("Unknown variable used in input: [" + pair.getFirst() + "]"); int nodeId = reverseMap.get(pair.getFirst()); int outputIndex = pair.getSecond();
for (i = 0; i < nInArrays; i++) { mergedFeatures[i] = pair.getFirst(); mergedFeaturesMasks[i] = pair.getSecond(); if (mergedFeaturesMasks[i] != null) needFeaturesMasks = true; for (i = 0; i < nOutArrays; i++) { Pair<INDArray, INDArray> pair = DataSetUtil.mergeLabels(labels, labelsMasks, i); mergedLabels[i] = pair.getFirst(); mergedLabelsMasks[i] = pair.getSecond(); if (mergedLabelsMasks[i] != null) needLabelsMasks = true;