public static MultiLayerNetwork train(WordVectors wordVectors, ObjectStream<NameSample> samples, int epochs, int windowSize, String[] labels) throws IOException { int vectorSize = 300; int layerSize = 256; MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .updater(new RmsProp(0.01)).l2(0.001) .weightInit(WeightInit.XAVIER) .list() .layer(0, new GravesLSTM.Builder().nIn(vectorSize).nOut(layerSize) .activation(Activation.TANH).build()) .layer(1, new RnnOutputLayer.Builder().activation(Activation.SOFTMAX) .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(layerSize).nOut(3).build()) .pretrain(false).backprop(true).build(); MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); net.setListeners(new ScoreIterationListener(5)); // TODO: Extract labels on the fly from the data DataSetIterator train = new NameSampleDataSetIterator(samples, wordVectors, windowSize, labels); System.out.println("Starting training"); for (int i = 0; i < epochs; i++) { net.fit(train); train.reset(); System.out.println(String.format("Finished epoch %d", i)); } return net; }
.optimizationAlgo(aTraits.getOptimizationAlgorithm()) .updater(new Nesterovs( new StepSchedule(ScheduleType.ITERATION, 1e-2, 0.1, 100000), 0.9))
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) // use stochastic gradient descent as an optimization algorithm .iterations(1) .activation(Activation.RELU)
public ComputationGraphConfiguration conf() { GraphBuilder graph = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations) .activation(Activation.RELU).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .learningRate(1e-2).biasLearningRate(2 * 1e-2).learningRateDecayPolicy(LearningRatePolicy.Step) .lrPolicyDecayRate(0.96).lrPolicySteps(320000).updater(new Nesterovs(1e-2, 0.9))
MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder() .seed(seed) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .weightInit(WeightInit.XAVIER) .updater(new Sgd(learningRate))
.seed(seed) .iterations(1) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .learningRate(learningRate).regularization(regularization).l2(regularizationRate) .updater(Updater.ADAGRAD)
public void buildModel() { //Create the network int numInput = 2; int numOutputs = 1; int nHidden = 10; mNetwork = new MultiLayerNetwork(new NeuralNetConfiguration.Builder() .seed(mSeed) .iterations(ITERATIONS) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .learningRate(LEARNING_RATE) .weightInit(WeightInit.XAVIER) .updater(Updater.NESTEROVS) .list() .layer(0, new DenseLayer.Builder().nIn(numInput).nOut(nHidden) .activation(Activation.TANH) .name("input") .build()) .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE) .activation(Activation.IDENTITY) .name("output") .nIn(nHidden).nOut(numOutputs).build()) .pretrain(false) .backprop(true) .build() ); mNetwork.init(); mNetwork.setListeners(mIterationListener); }
public MultiLayerConfiguration createNetwork() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .seed(seed) .iterations(1) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .learningRate(learningRate).regularization(regularization).l2(regularizationRate) .updater(Updater.ADAGRAD) .list() .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes) .weightInit(WEIGHT_INIT) .activation("relu").learningRateDecayPolicy(learningRatePolicy) .build()) .layer(1, new DenseLayer.Builder().nIn(numHiddenNodes).nOut(numHiddenNodes) .weightInit(WEIGHT_INIT) .activation("relu").learningRateDecayPolicy(learningRatePolicy) .build()) .layer(2, new DenseLayer.Builder().nIn(numHiddenNodes).nOut(numHiddenNodes) .weightInit(WEIGHT_INIT) .activation("relu").learningRateDecayPolicy(learningRatePolicy) .build()) .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MSE) .weightInit(WEIGHT_INIT) .activation("sigmoid").learningRateDecayPolicy(learningRatePolicy) .nIn(numHiddenNodes).nOut(numOutputs).build()) .pretrain(false).backprop(true).build(); return conf; } }
public static void main(String[] args) { List<Data> data = readFile(""); RegIterator trainIter = new RegIterator(data, 1, 5, 5); // 构建模型 MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .seed(1234) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .weightInit(WeightInit.XAVIER) .updater(new Nesterovs(0.01, 0.9)) .list().layer(0, new GravesLSTM.Builder().activation(Activation.TANH).nIn(1).nOut(32) .build()) .layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE) .activation(Activation.IDENTITY).nIn(32).nOut(1).build()) .build(); MultiLayerNetwork network = new MultiLayerNetwork(conf); network.setListeners(new ScoreIterationListener(1)); network.init(); int epoch = 10; for (int i = 0; i < epoch; i++) { while (trainIter.hasNext()) { DataSet dataSets = trainIter.next(); network.fit(dataSets); } trainIter.reset(); } }
/** * Deliver access to the internal builder * * @return NeuralNetworkConfiguration */ public org.deeplearning4j.nn.conf.NeuralNetConfiguration.Builder builder() { Builder builder = new Builder(); // Set dist to null if Disabled was chosen as dl4j backend defaults to null builder .l1(l1) .l2(l2) .optimizationAlgo(optimizationAlgo) .seed(seed) .weightInit(weightInit) .dist(dist.getBackend()) .biasInit(biasInit) .updater(updater.getBackend()) .biasUpdater(biasUpdater.getBackend()) .dropOut(dropout.getBackend()) .miniBatch(miniBatch) .minimize(minimize) .weightNoise(weightNoise.getBackend()) .gradientNormalization(gradientNormalization.getBackend()) .gradientNormalizationThreshold(gradientNormalizationThreshold) .inferenceWorkspaceMode(inferenceWorkspaceMode) .trainingWorkspaceMode(trainingWorkspaceMode); builder.setPretrain(pretrain); return builder; }
public MultiLayerConfiguration createNetwork() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .seed(seed) .iterations(1) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .learningRate(learningRate).regularization(regularization).l2(regularizationRate) .updater(Updater.ADAGRAD) .list() .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes) .weightInit(WEIGHT_INIT) .activation("relu").learningRateDecayPolicy(learningRatePolicy) .build()) .layer(1, new DenseLayer.Builder().nIn(numHiddenNodes).nOut(numHiddenNodes) .weightInit(WEIGHT_INIT) .activation("relu").learningRateDecayPolicy(learningRatePolicy) .build()) .layer(2, new OutputLayer.Builder(lossFunction) .weightInit(WEIGHT_INIT) .activation("softmax").learningRateDecayPolicy(learningRatePolicy) .nIn(numHiddenNodes).nOut(numOutputs).build()) .pretrain(false).backprop(true).build(); return conf; } }
public MultiLayerConfiguration conf() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().trainingWorkspaceMode(workspaceMode) .inferenceWorkspaceMode(workspaceMode).seed(seed).iterations(iterations) .activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new AdaDelta()) .regularization(false).convolutionMode(ConvolutionMode.Same).list() // block 1 .layer(0, new ConvolutionLayer.Builder(new int[] {5, 5}, new int[] {1, 1}).name("cnn1") .nIn(inputShape[0]).nOut(20).activation(Activation.RELU).build()) .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}, new int[] {2, 2}).name("maxpool1").build()) // block 2 .layer(2, new ConvolutionLayer.Builder(new int[] {5, 5}, new int[] {1, 1}).name("cnn2").nOut(50) .activation(Activation.RELU).build()) .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}, new int[] {2, 2}).name("maxpool2").build()) // fully connected .layer(4, new DenseLayer.Builder().name("ffn1").activation(Activation.RELU).nOut(500).build()) // output .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).name("output") .nOut(numLabels).activation(Activation.SOFTMAX) // radial basis function required .build()) .setInputType(InputType.convolutionalFlat(inputShape[2], inputShape[1], inputShape[0])) .backprop(true).pretrain(false).build(); return conf; }
public static void main(String[] args){ //Generate the training data DataSetIterator iterator = getTrainingData(batchSize,rng); //Create the network int numInput = 2; int numOutputs = 1; MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder() .seed(seed) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .weightInit(WeightInit.XAVIER) .updater(new Sgd(learningRate)) .list() .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MSE) .activation(Activation.IDENTITY) .nIn(numInput).nOut(numOutputs).build()) .pretrain(false).backprop(true).build() ); net.init(); net.setListeners(new ScoreIterationListener(1)); for( int i=0; i<nEpochs; i++ ){ iterator.reset(); net.fit(iterator); } final INDArray input = Nd4j.create(new double[] { 0.111111, 0.3333333333333 }, new int[] { 1, 2 }); INDArray out = net.output(input, false); System.out.println(out); }
public void initializeBuilder(String... inputNames) { if (inputNames.length == 0) { inputNames = new String[]{"input"}; } NeuralNetConfiguration.Builder graphBuilder = new NeuralNetConfiguration.Builder() .seed(args().seed) .iterations(1) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .learningRate(args().learningRate) .updater(Updater.ADAGRAD) .epsilon(BUILDER_EPSILON) .lrPolicyDecayRate(0.5) .weightInit(WEIGHT_INIT); if (args().regularizationRate != null) { graphBuilder.l2(args().regularizationRate); graphBuilder.regularization(args().regularizationRate != null); } if (args().dropoutRate != null) { graphBuilder.dropOut(args().dropoutRate); graphBuilder.setUseDropConnect(true); } modelCapacity=args().modelCapacity; reductionRate=args().reductionRate; build = graphBuilder.graphBuilder().addInputs(inputNames); }
/** Returns the network configuration, 2 hidden DenseLayers of size 50. */ private static MultiLayerConfiguration getDeepDenseLayerNetworkConfiguration() { final int numHiddenNodes = 100; return new NeuralNetConfiguration.Builder() .seed(seed) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .weightInit(WeightInit.XAVIER) .updater(new Nesterovs(learningRate, 0.9)) .list() .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes) .activation(Activation.RELU).build()) .layer(1, new DenseLayer.Builder().nIn(numHiddenNodes).nOut(numHiddenNodes) .activation(Activation.RELU).build()) .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MSE) .activation(Activation.IDENTITY) .nIn(numHiddenNodes).nOut(numOutputs).build()) .pretrain(false).backprop(true).build(); }
public MultiLayerConfiguration conf() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1) .learningRate(0.01).seed(12345).regularization(true).l2(0.001).weightInit(WeightInit.XAVIER) .updater(new RmsProp()).list() .layer(0, new GravesLSTM.Builder().nIn(inputShape[1]).nOut(256).activation(Activation.TANH) .build()) .layer(1, new GravesLSTM.Builder().nOut(256).activation(Activation.TANH).build()) .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) .activation(Activation.SOFTMAX) //MCXENT + softmax for classification .nOut(totalUniqueCharacters).build()) .backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(50).tBPTTBackwardLength(50) .pretrain(false).backprop(true).build(); return conf; }
public static MultiLayerNetwork lenetModel() { /** * Revisde Lenet Model approach developed by ramgo2 achieves slightly above random * Reference: https://gist.github.com/ramgo2/833f12e92359a2da9e5c2fb6333351c5 **/ MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .seed(seed) .l2(0.005) // tried 0.0001, 0.0005 .activation(Activation.RELU) .weightInit(WeightInit.XAVIER) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .updater(new Nesterovs(0.0001,0.9)) .list() .layer(0, new ConvolutionLayer.Builder(new int[]{5, 5}, new int[]{1, 1}, new int[]{0, 0}).name("cnn1") .nIn(channels).nOut(50).biasInit(0).build()) .layer(1, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool1").build()) .layer(2, new ConvolutionLayer.Builder(new int[]{5,5}, new int[]{5, 5}, new int[]{1, 1}).name("cnn2") .nOut(100).biasInit(0).build()) .layer(3, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool2").build()) .layer(4, new DenseLayer.Builder().nOut(500).build()) .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) .nOut(4) .activation(Activation.SOFTMAX) .build()) .backprop(true).pretrain(false) .setInputType(InputType.convolutional(height, width, channels)) .build(); return new MultiLayerNetwork(conf); }
.learningRate(learningRate) .weightInit(WeightInit.XAVIER) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .regularization(true) .l2(lambdaReg)
.learningRate(learningRate) .weightInit(WeightInit.XAVIER_UNIFORM) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .regularization(true) .l2(lambdaReg)
public static MultiLayerConfiguration lenetModelConf() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .seed(seed) .l2(0.005) .activation(Activation.RELU) .weightInit(WeightInit.XAVIER) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .updater(new Nesterovs(0.0001, 0.9)) .list() .layer(0, new ConvolutionLayer.Builder(new int[]{5, 5}, new int[]{1, 1}, new int[]{0, 0}).name("cnn1") .nIn(channels).nOut(50).biasInit(0).build()) .layer(1, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool1").build()) .layer(2, new ConvolutionLayer.Builder(new int[]{5,5}, new int[]{5, 5}, new int[]{1, 1}).name("cnn2") .nOut(100).biasInit(0).build()) .layer(3, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool2").build()) .layer(4, new DenseLayer.Builder().nOut(500).build()) .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) .nOut(4) .activation(Activation.SOFTMAX) .build()) .backprop(true).pretrain(false) .setInputType(InputType.convolutional(height, width, channels)) .build(); return conf; } public static void saveModel(FileSystem fs, Model model ) throws Exception{