@Override public void exec(int... dimensions) { super.exec(dimensions); z.muli(z.rsub(1)); }
INDArray secondTerm = output.rsubi(1); Transforms.log(secondTerm, false); secondTerm.muli(labels.rsub(1)); scoreArr.addi(secondTerm);
private INDArray calcLogProbArray(INDArray x, INDArray preOutDistributionParams) { INDArray output = preOutDistributionParams.dup(); activationFn.getActivation(output, false); INDArray logOutput = Transforms.log(output, true); INDArray log1SubOut = Transforms.log(output.rsubi(1.0), false); //For numerical stability: if output = 0, then log(output) == -infinity //then x * log(output) = NaN, but lim(x->0, output->0)[ x * log(output) ] == 0 // therefore: want 0*log(0) = 0, NOT 0*log(0) = NaN by default BooleanIndexing.replaceWhere(logOutput, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0 BooleanIndexing.replaceWhere(log1SubOut, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0 return logOutput.muli(x).addi(x.rsub(1.0).muli(log1SubOut)); }
isPositiveLabel = labels; isNegativeLabel = Transforms.not(isPositiveLabel); pClass0 = output.rsub(1.0); pClass1 = output; } else {
INDArray secondTerm = output.rsubi(1); Transforms.log(secondTerm, false); secondTerm.muli(labels.rsub(1)); scoreArr.addi(secondTerm);
@Override public void exec(int... dimensions) { super.exec(dimensions); z.muli(z.rsub(1)); }
case RECONSTRUCTION_CROSSENTROPY: INDArray xEntLogZ2 = logZ(z); INDArray xEntOneMinusLabelsOut2 = labels.rsub(1); INDArray xEntOneMinusLogOneMinusZ2 = xEntLogZ2.rsubi(1); INDArray temp = labels.mul(xEntLogZ2).add(xEntOneMinusLabelsOut2).muli(xEntOneMinusLogOneMinusZ2); case XENT: INDArray xEntLogZ = logZ(z); INDArray xEntOneMinusLabelsOut = labels.rsub(1); INDArray xEntOneMinusLogOneMinusZ = xEntLogZ.dup().rsubi(1); INDArray temp2 = labels.mul(xEntLogZ).add(xEntOneMinusLabelsOut).muli(xEntOneMinusLogOneMinusZ);
isPositiveLabel = labels; isNegativeLabel = Transforms.not(isPositiveLabel); pClass0 = output.rsub(1.0); pClass1 = output; } else {
@Override public INDArray gradient(INDArray x, INDArray preOutDistributionParams) { INDArray output = preOutDistributionParams.dup(); activationFn.getActivation(output, true); INDArray diff = x.sub(output); INDArray outOneMinusOut = output.rsub(1.0).muli(output); INDArray grad = diff.divi(outOneMinusOut); grad = activationFn.backprop(preOutDistributionParams.dup(), grad).getFirst(); //Issue: if output == 0 or output == 1, then (assuming sigmoid output or similar) //sigmaPrime == 0, sigmaPrime * (x-out) / (out*(1-out)) == 0 * (x-out) / 0 -> 0/0 -> NaN. But taking limit, we want //0*(x-out)/0 == 0 -> implies 0 gradient at the far extremes (0 or 1) of the output BooleanIndexing.replaceWhere(grad, 0.0, Conditions.isNan()); return grad.negi(); }
negativeActualClassColumn = labels.rsub(1.0); //1.0 - label positivePredictedClassColumn = predictions; } else {
@Override public void computeGradientAndScore() { INDArray W = getParam(PretrainParamInitializer.WEIGHT_KEY); double corruptionLevel = layerConf().getCorruptionLevel(); INDArray corruptedX = corruptionLevel > 0 ? getCorruptedInput(input, corruptionLevel) : input; setInput(corruptedX); INDArray y = encode(corruptedX, true); INDArray z = decode(y); INDArray visibleLoss = input.sub(z); INDArray hiddenLoss = layerConf().getSparsity() == 0 ? visibleLoss.mmul(W).muli(y).muli(y.rsub(1)) : visibleLoss.mmul(W).muli(y).muli(y.add(-layerConf().getSparsity())); INDArray wGradient = corruptedX.transposei().mmul(hiddenLoss).addi(visibleLoss.transposei().mmul(y)); INDArray hBiasGradient = hiddenLoss.sum(0); INDArray vBiasGradient = visibleLoss.sum(0); gradient = createGradient(wGradient, vBiasGradient, hBiasGradient); setScoreWithZ(z); }
INDArray sorted = Nd4j.sortRows(pl, 0, false); INDArray isPositive = sorted.getColumn(1); INDArray isNegative = sorted.getColumn(1).rsub(1.0);
hBiasGradient = probHidden.getFirst().rsub(layerConf().getSparsity()).sum(0); else