In [6]:
import $ivy.`org.deeplearning4j:deeplearning4j-core:0.7.2`
import $ivy.`org.nd4j:nd4j-native-platform:0.7.2`

import org.nd4j.linalg.activations.Activation
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator
import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator
import org.deeplearning4j.eval.Evaluation
import org.deeplearning4j.nn.api.OptimizationAlgorithm
import org.deeplearning4j.nn.conf.MultiLayerConfiguration
import org.deeplearning4j.nn.conf.NeuralNetConfiguration
import org.deeplearning4j.nn.conf.Updater
import org.deeplearning4j.nn.conf.layers.DenseLayer
import org.deeplearning4j.nn.conf.layers.OutputLayer
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork
import org.deeplearning4j.nn.weights.WeightInit
import org.deeplearning4j.optimize.listeners.ScoreIterationListener
import org.nd4j.linalg.api.ndarray.INDArray
import org.nd4j.linalg.dataset.DataSet
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction
import org.slf4j.Logger
import org.slf4j.LoggerFactory


Out[6]:
import $ivy.$                                             

import $ivy.$                                    


import org.nd4j.linalg.activations.Activation

import org.nd4j.linalg.dataset.api.iterator.DataSetIterator

import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator

import org.deeplearning4j.eval.Evaluation

import org.deeplearning4j.nn.api.OptimizationAlgorithm

import org.deeplearning4j.nn.conf.MultiLayerConfiguration

import org.deeplearning4j.nn.conf.NeuralNetConfiguration

import org.deeplearning4j.nn.conf.Updater

import org.deeplearning4j.nn.conf.layers.DenseLayer

import org.deeplearning4j.nn.conf.layers.OutputLayer

import org.deeplearning4j.nn.multilayer.MultiLayerNetwork

import org.deeplearning4j.nn.weights.WeightInit

import org.deeplearning4j.optimize.listeners.ScoreIterationListener

import org.nd4j.linalg.api.ndarray.INDArray

import org.nd4j.linalg.dataset.DataSet

import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction

import org.slf4j.Logger

import org.slf4j.LoggerFactory

In [2]:
//number of rows and columns in the input pictures
val numRows = 28;
val numColumns = 28;
val outputNum = 10; // number of output classes
val batchSize = 64; // batch size for each epoch
val rngSeed = 123; // random number seed for reproducibility
val numEpochs = 15; // number of epochs to perform
val rate = 0.0015; // learning rate

        //Get the DataSetIterators:
val mnistTrain = new MnistDataSetIterator(batchSize, true, rngSeed);
val mnistTest = new MnistDataSetIterator(batchSize, false, rngSeed);


SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.

Out[2]:
numRows: Int = 28
numColumns: Int = 28
outputNum: Int = 10
batchSize: Int = 64
rngSeed: Int = 123
numEpochs: Int = 15
rate: Double = 0.0015
mnistTrain: MnistDataSetIterator = org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator@6328fce2
mnistTest: MnistDataSetIterator = org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator@69fa20b6

Create MLP model:


In [3]:
val builder = new NeuralNetConfiguration.Builder()

builder
  .seed(rngSeed) //include a random seed for reproducibility
  .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) // use stochastic gradient descent as an optimization algorithm
  .iterations(1)
  .activation(Activation.RELU)
  .weightInit(WeightInit.XAVIER)
  .learningRate(rate) //specify the learning rate
  .updater(Updater.NESTEROVS).momentum(0.98) //specify the rate of change of the learning rate.
  .regularization(true).l2(rate * 0.005) // regularize learning model

val lstBuilder = builder.list()

lstBuilder.layer(0, new DenseLayer.Builder().nIn(numRows * numColumns).nOut(500).build())
lstBuilder.layer(1, new DenseLayer.Builder().nIn(500).nOut(100).build())
lstBuilder.layer(2, new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD) 
                    .activation(Activation.SOFTMAX)
                    .nIn(100)
                    .nOut(outputNum)
                    .build())

lstBuilder.pretrain(false)
lstBuilder.backprop(true)

val conf = lstBuilder.build()

val model = new MultiLayerNetwork(conf)
model.init()
model.setListeners(new ScoreIterationListener(5))  //print the score with every iteration


Out[3]:
builder: NeuralNetConfiguration.Builder = NeuralNetConfiguration.Builder(activationFn=relu, weightInit=XAVIER, biasInit=0.0, dist=null, learningRate=0.0015, biasLearningRate=NaN, learningRateSchedule=null, lrScoreBasedDecay=0.0, l1=NaN, l2=7.5E-6, dropOut=0.0, updater=NESTEROVS, momentum=0.98, momentumSchedule=null, epsilon=NaN, rho=NaN, rmsDecay=NaN, adamMeanDecay=NaN, adamVarDecay=NaN, layer=null, leakyreluAlpha=0.01, miniBatch=true, numIterations=1, maxNumLineSearchIterations=5, seed=123, useRegularization=true, optimizationAlgo=STOCHASTIC_GRADIENT_DESCENT, stepFunction=null, useDropConnect=false, minimize=true, gradientNormalization=None, gradientNormalizationThreshold=1.0, learningRatePolicy=None, lrPolicyDecayRate=NaN, lrPolicySteps=NaN, lrPolicyPower=NaN, pretrain=false, convolutionMode=Truncate)
res2_1: NeuralNetConfiguration.Builder = NeuralNetConfiguration.Builder(activationFn=relu, weightInit=XAVIER, biasInit=0.0, dist=null, learningRate=0.0015, biasLearningRate=NaN, learningRateSchedule=null, lrScoreBasedDecay=0.0, l1=NaN, l2=7.5E-6, dropOut=0.0, updater=NESTEROVS, momentum=0.98, momentumSchedule=null, epsilon=NaN, rho=NaN, rmsDecay=NaN, adamMeanDecay=NaN, adamVarDecay=NaN, layer=null, leakyreluAlpha=0.01, miniBatch=true, numIterations=1, maxNumLineSearchIterations=5, seed=123, useRegularization=true, optimizationAlgo=STOCHASTIC_GRADIENT_DESCENT, stepFunction=null, useDropConnect=false, minimize=true, gradientNormalization=None, gradientNormalizationThreshold=1.0, learningRatePolicy=None, lrPolicyDecayRate=NaN, lrPolicySteps=NaN, lrPolicyPower=NaN, pretrain=false, convolutionMode=Truncate)
lstBuilder: NeuralNetConfiguration.ListBuilder = MultiLayerConfiguration.Builder(confs=[], dampingFactor=100.0, inputPreProcessors={}, pretrain=false, backprop=true, backpropType=Standard, tbpttFwdLength=20, tbpttBackLength=20, inputType=null, cnnInputSize=null)
res2_3: NeuralNetConfiguration.ListBuilder = MultiLayerConfiguration.Builder(confs=[], dampingFactor=100.0, inputPreProcessors={}, pretrain=false, backprop=true, backpropType=Standard, tbpttFwdLength=20, tbpttBackLength=20, inputType=null, cnnInputSize=null)
res2_4: NeuralNetConfiguration.ListBuilder = MultiLayerConfiguration.Builder(confs=[], dampingFactor=100.0, inputPreProcessors={}, pretrain=false, backprop=true, backpropType=Standard, tbpttFwdLength=20, tbpttBackLength=20, inputType=null, cnnInputSize=null)
res2_5: NeuralNetConfiguration.ListBuilder = MultiLayerConfiguration.Builder(confs=[], dampingFactor=100.0, inputPreProcessors={}, pretrain=false, backprop=true, backpropType=Standard, tbpttFwdLength=20, tbpttBackLength=20, inputType=null, cnnInputSize=null)
res2_6: NeuralNetConfiguration.ListBuilder = MultiLayerConfiguration.Builder(confs=[], dampingFactor=100.0, inputPreProcessors={}, pretrain=false, backprop=true, backpropType=Standard, tbpttFwdLength=20, tbpttBackLength=20, inputType=null, cnnInputSize=null)
res2_7: NeuralNetConfiguration.ListBuilder = MultiLayerConfiguration.Builder(confs=[], dampingFactor=100.0, inputPreProcessors={}, pretrain=false, backprop=true, backpropType=Standard, tbpttFwdLength=20, tbpttBackLength=20, inputType=null, cnnInputSize=null)
conf: MultiLayerConfiguration = {
  "backprop" : true,
  "backpropType" : "Standard",
  "confs" : [ {
    "iterationCount" : 0,
    "l1ByParam" : {
      "b" : 0.0,
      "W" : 0.0
    },
    "l2ByParam" : {
      "b" : 0.0,
      "W" : 7.5E-6
...
model: MultiLayerNetwork = org.deeplearning4j.nn.multilayer.MultiLayerNetwork@4829e0b4

Train model:


In [4]:
println("Train model....");
(0 to numEpochs).foreach{i =>
  println("Epoch " + i)
  model.fit(mnistTrain)
}


Train model....
Epoch 0
Epoch 1
Epoch 2
Epoch 3
Epoch 4
Epoch 5
Epoch 6
Epoch 7
Epoch 8
Epoch 9
Epoch 10
Epoch 11
Epoch 12
Epoch 13
Epoch 14
Epoch 15

Evaluate model:


In [5]:
val eval = new Evaluation(outputNum) //create an evaluation object with 10 possible classes

while(mnistTest.hasNext()){
  val next = mnistTest.next()
  val output = model.output(next.getFeatureMatrix()) //get the networks prediction
  eval.eval(next.getLabels(), output) //check the prediction against the true class
}

println(eval.stats());
println("****************Example finished********************");


Examples labeled as 0 classified by model as 0: 971 times
Examples labeled as 0 classified by model as 2: 1 times
Examples labeled as 0 classified by model as 3: 1 times
Examples labeled as 0 classified by model as 6: 1 times
Examples labeled as 0 classified by model as 7: 1 times
Examples labeled as 0 classified by model as 8: 4 times
Examples labeled as 0 classified by model as 9: 1 times
Examples labeled as 1 classified by model as 1: 1128 times
Examples labeled as 1 classified by model as 2: 2 times
Examples labeled as 1 classified by model as 3: 1 times
Examples labeled as 1 classified by model as 5: 1 times
Examples labeled as 1 classified by model as 6: 2 times
Examples labeled as 1 classified by model as 8: 1 times
Examples labeled as 2 classified by model as 0: 2 times
Examples labeled as 2 classified by model as 1: 2 times
Examples labeled as 2 classified by model as 2: 1015 times
Examples labeled as 2 classified by model as 3: 1 times
Examples labeled as 2 classified by model as 4: 3 times
Examples labeled as 2 classified by model as 6: 3 times
Examples labeled as 2 classified by model as 7: 2 times
Examples labeled as 2 classified by model as 8: 4 times
Examples labeled as 3 classified by model as 0: 1 times
Examples labeled as 3 classified by model as 2: 4 times
Examples labeled as 3 classified by model as 3: 990 times
Examples labeled as 3 classified by model as 5: 3 times
Examples labeled as 3 classified by model as 7: 4 times
Examples labeled as 3 classified by model as 8: 3 times
Examples labeled as 3 classified by model as 9: 5 times
Examples labeled as 4 classified by model as 2: 3 times
Examples labeled as 4 classified by model as 4: 968 times
Examples labeled as 4 classified by model as 5: 1 times
Examples labeled as 4 classified by model as 6: 2 times
Examples labeled as 4 classified by model as 9: 8 times
Examples labeled as 5 classified by model as 0: 2 times
Examples labeled as 5 classified by model as 3: 5 times
Examples labeled as 5 classified by model as 4: 2 times
Examples labeled as 5 classified by model as 5: 875 times
Examples labeled as 5 classified by model as 6: 4 times
Examples labeled as 5 classified by model as 7: 1 times
Examples labeled as 5 classified by model as 8: 2 times
Examples labeled as 5 classified by model as 9: 1 times
Examples labeled as 6 classified by model as 0: 3 times
Examples labeled as 6 classified by model as 1: 3 times
Examples labeled as 6 classified by model as 2: 2 times
Examples labeled as 6 classified by model as 3: 1 times
Examples labeled as 6 classified by model as 4: 9 times
Examples labeled as 6 classified by model as 5: 4 times
Examples labeled as 6 classified by model as 6: 936 times
Examples labeled as 7 classified by model as 0: 1 times
Examples labeled as 7 classified by model as 1: 6 times
Examples labeled as 7 classified by model as 2: 7 times
Examples labeled as 7 classified by model as 3: 1 times
Examples labeled as 7 classified by model as 4: 3 times
Examples labeled as 7 classified by model as 7: 1000 times
Examples labeled as 7 classified by model as 8: 6 times
Examples labeled as 7 classified by model as 9: 4 times
Examples labeled as 8 classified by model as 0: 4 times
Examples labeled as 8 classified by model as 2: 4 times
Examples labeled as 8 classified by model as 3: 3 times
Examples labeled as 8 classified by model as 4: 3 times
Examples labeled as 8 classified by model as 5: 5 times
Examples labeled as 8 classified by model as 6: 2 times
Examples labeled as 8 classified by model as 7: 3 times
Examples labeled as 8 classified by model as 8: 948 times
Examples labeled as 8 classified by model as 9: 2 times
Examples labeled as 9 classified by model as 0: 3 times
Examples labeled as 9 classified by model as 1: 2 times
Examples labeled as 9 classified by model as 3: 8 times
Examples labeled as 9 classified by model as 4: 10 times
Examples labeled as 9 classified by model as 5: 4 times
Examples labeled as 9 classified by model as 7: 3 times
Examples labeled as 9 classified by model as 9: 979 times


==========================Scores========================================
 Accuracy:        0.981
 Precision:       0.9809
 Recall:          0.9808
 F1 Score:        0.9809
========================================================================
****************Example finished********************
Out[5]:
eval: Evaluation = org.deeplearning4j.eval.Evaluation@6c85a02d

In [ ]: