public static void main(String[] args) { //Nd4j.dtype = DataBuffer.DOUBLE; INDArray nd1 = Nd4j.create(new double[]{1,2,3,4,5,6},new int[]{2,3}); System.out.println("nd1:\n"+nd1); //create nd-array variable ndv to be able to print result of nondestructive operations. add scalar to matrix and assign ndv the sum. INDArray ndv = nd1.add(1); System.out.println("nd1.add(1):\n"+ndv); ndv = nd1.mul(5); System.out.println("nd1.mul(5):\n"+ndv); ndv = nd1.sub(3); System.out.println("nd1.sub(3):\n"+ndv); ndv = nd1.div(2); System.out.println("nd1.div(2):\n"+ndv); //add column vector to matrix INDArray nd2 = Nd4j.create(new double[]{10,20},new int[]{2,1}); //vector as column System.out.println("nd2:\n"+nd2); ndv = nd1.addColumnVector(nd2); System.out.println("nd1.addColumnVector(nd2):\n"+ndv); // add row vector to matrix INDArray nd3 = Nd4j.create(new double[]{30,40,50},new int[]{1, 3}); //vector as row System.out.println("nd3:\n"+nd3); ndv = nd1.addRowVector(nd3); System.out.println("nd1.addRowVector(nd3):\n"+ndv); //multiply two matrices of equal dimensions elementwise. INDArray nd4 = Nd4j.create(new double[]{1,2,1,2,1,2},new int[]{2,3}); System.out.println("nd4:\n"+nd4); ndv = nd1.mul(nd4); System.out.println("nd1.mul(nd4):\n"+ndv); }
private INDArray calculateBernoulli(INDArray minorityLabels, INDArray labelMask, double targetMinorityDist) { INDArray minorityClass = minorityLabels.dup().muli(labelMask); INDArray majorityClass = Transforms.not(minorityLabels).muli(labelMask); //all minorityLabel class, keep masks as is //presence of minoriy class and donotmask minority windows set to true return label as is if (majorityClass.sumNumber().intValue() == 0 || (minorityClass.sumNumber().intValue() > 0 && donotMaskMinorityWindows)) return labelMask; //all majority class and set to not mask all majority windows sample majority class by 1-targetMinorityDist if (minorityClass.sumNumber().intValue() == 0 && !maskAllMajorityWindows) return labelMask.muli(1 - targetMinorityDist); //Probabilities to be used for bernoulli sampling INDArray minoritymajorityRatio = minorityClass.sum(1).div(majorityClass.sum(1)); INDArray majorityBernoulliP = minoritymajorityRatio.muli(1 - targetMinorityDist).divi(targetMinorityDist); BooleanIndexing.replaceWhere(majorityBernoulliP, 1.0, Conditions.greaterThan(1.0)); //if minority ratio is already met round down to 1.0 return majorityClass.muliColumnVector(majorityBernoulliP).addi(minorityClass); }
INDArray normalPart = mdc.alpha.div(Transforms.pow(mdc.sigma.mul(SQRT_TWO_PI), mLabelWidth)); INDArray exponent = labelsMinusMuSquared.div(minustwovariance); INDArray exponentMax = exponent.max(1); exponent.subiColumnVector(exponentMax); INDArray dLdZSigma = (labelsMinusMuSquared.div(variance).subi(mLabelWidth)).muli(-1).muli(pi);
@Override public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } //INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup())); INDArray output = activationFn.getActivation(preOutput.dup(), true); INDArray dLda = labels.div(output).negi(); if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) { //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j) //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be // error prone - though buy us a tiny bit of performance LossUtil.applyMask(dLda, mask); } INDArray grad = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params if (mask != null) { LossUtil.applyMask(grad, mask); } return grad; }
private INDArray calculateBernoulli(INDArray minorityLabels, INDArray labelMask, double targetMinorityDist) { INDArray minorityClass = minorityLabels.dup().muli(labelMask); INDArray majorityClass = Transforms.not(minorityLabels).muli(labelMask); //all minorityLabel class, keep masks as is //presence of minoriy class and donotmask minority windows set to true return label as is if (majorityClass.sumNumber().intValue() == 0 || (minorityClass.sumNumber().intValue() > 0 && donotMaskMinorityWindows)) return labelMask; //all majority class and set to not mask all majority windows sample majority class by 1-targetMinorityDist if (minorityClass.sumNumber().intValue() == 0 && !maskAllMajorityWindows) return labelMask.muli(1-targetMinorityDist); //Probabilities to be used for bernoulli sampling INDArray minoritymajorityRatio = minorityClass.sum(1).div(majorityClass.sum(1)); INDArray majorityBernoulliP = minoritymajorityRatio.muli(1 - targetMinorityDist).divi(targetMinorityDist); BooleanIndexing.replaceWhere(majorityBernoulliP,1.0, Conditions.greaterThan(1.0)); //if minority ratio is already met round down to 1.0 return majorityClass.muliColumnVector(majorityBernoulliP).addi(minorityClass); }
@Override public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray yHat = activationFn.getActivation(preOutput.dup(), true); INDArray yDivyhat = labels.div(yHat); INDArray dLda = yDivyhat.rsubi(1); if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) { //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j) //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be // error prone - though buy us a tiny bit of performance LossUtil.applyMask(dLda, mask); } INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params if (mask != null) { LossUtil.applyMask(gradients, mask); } return gradients; }
public static boolean checkDivManually(INDArray first, INDArray second, double maxRelativeDifference, double minAbsDifference) { //No apache commons element-wise division, but can do this manually INDArray result = first.div(second); long[] shape = first.shape(); INDArray expected = Nd4j.zeros(first.shape()); for (int i = 0; i < shape[0]; i++) { for (int j = 0; j < shape[1]; j++) { double v = first.getDouble(i, j) / second.getDouble(i, j); expected.putScalar(new int[] {i, j}, v); } } if (!checkShape(expected, result)) return false; boolean ok = checkEntries(expected, result, maxRelativeDifference, minAbsDifference); if (!ok) { INDArray onCopies = Shape.toOffsetZeroCopy(first).mul(Shape.toOffsetZeroCopy(second)); printFailureDetails(first, second, expected, result, onCopies, "div"); } return ok; }
/** * * @param func * @param x0 * @param f0 * @param h * @param oneSided * @return */ public static INDArray denseDifference(Function<INDArray,INDArray> func, INDArray x0,INDArray f0, INDArray h,INDArray oneSided) { INDArray hVecs = Nd4j.diag(h.reshape(1,h.length())); INDArray dx,df,x; INDArray jTransposed = Nd4j.create(x0.length(),f0.length()); for(int i = 0; i < h.length(); i++) { INDArray hVecI = hVecs.slice(i); x = (x0.add(hVecI)); dx = x.slice(i).sub(x0.slice(i)); df = func.apply(x).sub(f0); INDArray div = df.div(dx); jTransposed.putSlice(i,div); } if(f0.length() == 1) jTransposed = jTransposed.ravel(); return jTransposed; }
/** * Pre process image to reduce to make it feedable to inception network * * @param input Input image * @return processed image */ public INDArray preProcessImage(INDArray input) { // Transform to [-1.0, 1.0] range return input.div(255.0).sub(0.5).mul(2.0); }
protected INDArray normalizeIfNeeded(INDArray image){ return image.div(normalizeValue); }
public StackedRNN(float learningRate, int seqLength, int hiddenLayerSize, int epochs, String text, int batch, boolean useChars, boolean rmsProp) { super(learningRate, seqLength, hiddenLayerSize, epochs, text, batch, useChars); this.rmsProp = rmsProp; wxh = Nd4j.randn(hiddenLayerSize, vocabSize).div(Math.sqrt(hiddenLayerSize)); whh = Nd4j.randn(hiddenLayerSize, hiddenLayerSize).div(Math.sqrt(hiddenLayerSize)); whh2 = Nd4j.randn(hiddenLayerSize, hiddenLayerSize).div(Math.sqrt(hiddenLayerSize)); wxh2 = Nd4j.randn(hiddenLayerSize, hiddenLayerSize).div(Math.sqrt(hiddenLayerSize)); wh2y = Nd4j.randn(vocabSize, hiddenLayerSize).div(Math.sqrt(vocabSize)); bh = Nd4j.zeros(hiddenLayerSize, 1); bh2 = Nd4j.zeros(hiddenLayerSize, 1); by = Nd4j.zeros(vocabSize, 1); }
@Override public Job aggregate() { if(averaged == null) return empty(); Job ret = new Job(averaged.div(seenSoFar),""); seenSoFar = 0.0; return ret; }
@Override public INDArray getFunctionValues(final INDArray x) { return Nd4j.getExecutioner().execAndReturn(new Sin(x.dup())).div(x); }
/** * Encodes the word. Returns nullvector if word was not found. * @param word * @return */ @Override public INDArray encode(String word) { INDArray sum = Nd4j.zeros(getEmbeddingVectorSize(), 1); int len = 0; for(String w : WordHelpers.splitSpaces(word)) { if(w.trim().isEmpty()) continue; INDArray arr = vec.getWordVectorMatrix(preprocessor.preProcess(w)); if(arr != null) sum.addi(arr.transpose()); len++; } return len == 0 ? sum : sum.div(len); }
@Override public double calculateScore(MultiLayerNetwork network) { dataSetIterator.reset(); double losSum = 0.0; int exCount = 0; while (dataSetIterator.hasNext()) { DataSet dataSet = dataSetIterator.next(); if (dataSet == null) { break; } long nEx = dataSet.getFeatures().size(0); INDArray output = network.output(dataSet.getFeatures(), false); INDArray labels = dataSet.getLabels(); INDArray score = Transforms.abs(output.sub(labels)); score = score.div(labels); exCount += nEx; losSum += score.sumNumber().doubleValue(); } if (average) { return losSum / exCount; } return losSum; }
@Override public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException("Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray yHat = activationFn.getActivation(preOutput.dup(), true); INDArray yDivyhat = labels.div(yHat); INDArray dLda = yDivyhat.rsubi(1); if(mask != null && LossUtil.isPerOutputMasking(dLda, mask)){ //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j) //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be // error prone - though buy us a tiny bit of performance LossUtil.applyMask(dLda, mask); } INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params if (mask != null) { LossUtil.applyMask(gradients, mask); } return gradients; }
@Override public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException("Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } //INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup())); INDArray output = activationFn.getActivation(preOutput.dup(), true); INDArray dLda = labels.div(output).negi(); if(mask != null && LossUtil.isPerOutputMasking(dLda, mask)){ //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j) //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be // error prone - though buy us a tiny bit of performance LossUtil.applyMask(dLda, mask); } INDArray grad = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params if (mask != null) { LossUtil.applyMask(grad, mask); } return grad; }
public static boolean checkDivManually(INDArray first, INDArray second, double maxRelativeDifference, double minAbsDifference) { //No apache commons element-wise division, but can do this manually INDArray result = first.div(second); int[] shape = first.shape(); INDArray expected = Nd4j.zeros(first.shape()); for (int i = 0; i < shape[0]; i++) { for (int j = 0; j < shape[1]; j++) { double v = first.getDouble(i, j) / second.getDouble(i, j); expected.putScalar(new int[] {i, j}, v); } } if (!checkShape(expected, result)) return false; boolean ok = checkEntries(expected, result, maxRelativeDifference, minAbsDifference); if (!ok) { INDArray onCopies = Shape.toOffsetZeroCopy(first).mul(Shape.toOffsetZeroCopy(second)); printFailureDetails(first, second, expected, result, onCopies, "div"); } return ok; }