In [2]:
import numpy as np
Module is an abstract class which defines fundamental methods necessary for a training a neural network. You do not need to change anything here, just read the comments.
In [3]:
class Module(object):
"""
Basically, you can think of a module as of a something (black box)
which can process `input` data and produce `ouput` data.
This is like applying a function which is called `forward`:
output = module.forward(input)
The module should be able to perform a backward pass: to differentiate the `forward` function.
More, it should be able to differentiate it if is a part of chain (chain rule).
The latter implies there is a gradient from previous step of a chain rule.
gradInput = module.backward(input, gradOutput)
"""
def __init__ (self):
self.output = None
self.gradInput = None
self.training = True
def forward(self, input):
"""
Takes an input object, and computes the corresponding output of the module.
"""
return self.updateOutput(input)
def backward(self,input, gradOutput):
"""
Performs a backpropagation step through the module, with respect to the given input.
This includes
- computing a gradient w.r.t. `input` (is needed for further backprop),
- computing a gradient w.r.t. parameters (to update parameters while optimizing).
"""
self.updateGradInput(input, gradOutput)
self.accGradParameters(input, gradOutput)
return self.gradInput
def updateOutput(self, input):
"""
Computes the output using the current parameter set of the class and input.
This function returns the result which is stored in the `output` field.
Make sure to both store the data in `output` field and return it.
"""
# The easiest case:
# self.output = input
# return self.output
pass
def updateGradInput(self, input, gradOutput):
"""
Computing the gradient of the module with respect to its own input.
This is returned in `gradInput`. Also, the `gradInput` state variable is updated accordingly.
The shape of `gradInput` is always the same as the shape of `input`.
Make sure to both store the gradients in `gradInput` field and return it.
"""
# The easiest case:
# self.gradInput = gradOutput
# return self.gradInput
pass
def accGradParameters(self, input, gradOutput):
"""
Computing the gradient of the module with respect to its own parameters.
No need to override if module has no parameters (e.g. ReLU).
"""
pass
def zeroGradParameters(self):
"""
Zeroes `gradParams` variable if the module has params.
"""
pass
def getParameters(self):
"""
Returns a list with its parameters.
If the module does not have parameters return empty list.
"""
return []
def getGradParameters(self):
"""
Returns a list with gradients with respect to its parameters.
If the module does not have parameters return empty list.
"""
return []
def train(self):
"""
Sets training mode for the module.
Training and testing behaviour differs for Dropout, BatchNorm.
"""
self.training = True
def evaluate(self):
"""
Sets evaluation mode for the module.
Training and testing behaviour differs for Dropout, BatchNorm.
"""
self.training = False
def __repr__(self):
"""
Pretty printing. Should be overrided in every module if you want
to have readable description.
"""
return "Module"
Define a forward and backward pass procedures.
In [4]:
class Sequential(Module):
"""
This class implements a container, which processes `input` data sequentially.
`input` is processed by each module (layer) in self.modules consecutively.
The resulting array is called `output`.
"""
def __init__ (self):
super(Sequential, self).__init__()
self.modules = []
self.inputs = []
def add(self, module):
"""
Adds a module to the container.
"""
self.modules.append(module)
def updateOutput(self, input):
"""
Basic workflow of FORWARD PASS:
y_0 = module[0].forward(input)
y_1 = module[1].forward(y_0)
...
output = module[n-1].forward(y_{n-2})
Just write a little loop.
"""
# Your code goes here. ################################################
self.inputs = []
for module in self.modules:
output = module.forward(input)
input = output
self.inputs.append(input)
self.output = output
del self.inputs[-1]
return self.output
def backward(self, input, gradOutput):
"""
Workflow of BACKWARD PASS:
g_{n-1} = module[n-1].backward(y_{n-2}, gradOutput)
g_{n-2} = module[n-2].backward(y_{n-3}, g_{n-1})
...
g_1 = module[1].backward(y_0, g_2)
gradInput = module[0].backward(input, g_1)
!!!
To each module you need to provide the input, module saw while forward pass,
it is used while computing gradients.
Make sure that the input for `i-th` layer the output of `module[i]` (just the same input as in forward pass)
and NOT `input` to this Sequential module.
!!!
"""
# Your code goes here. ################################################
inputs = [input] + self.inputs.copy();
for module, forward_input in zip(reversed(self.modules), reversed(inputs)):
gradInput = module.backward(forward_input, gradOutput)
gradOutput = gradInput
self.gradInput = gradInput
return self.gradInput
def zeroGradParameters(self):
for module in self.modules:
module.zeroGradParameters()
def getParameters(self):
"""
Should gather all parameters in a list.
"""
return [x.getParameters() for x in self.modules]
def getGradParameters(self):
"""
Should gather all gradients w.r.t parameters in a list.
"""
return [x.getGradParameters() for x in self.modules]
def __repr__(self):
string = "".join([str(x) + '\n' for x in self.modules])
return string
def __getitem__(self,x):
return self.modules.__getitem__(x)
def train(self):
"""
Propagates training parameter through all modules
"""
self.training = True
for module in self.modules:
module.train()
def evaluate(self):
"""
Propagates training parameter through all modules
"""
self.training = False
for module in self.modules:
module.evaluate()
In [197]:
class Linear(Module):
"""
A module which applies a linear transformation
A common name is fully-connected layer, InnerProductLayer in caffe.
The module should work with 2D input of shape (n_samples, n_feature).
"""
def __init__(self, n_in, n_out):
super(Linear, self).__init__()
# This is a nice initialization
stdv = 1./np.sqrt(n_in)
self.W = np.random.uniform(-stdv, stdv, size = (n_out, n_in))
self.b = np.random.uniform(-stdv, stdv, size = n_out)
self.gradW = np.zeros_like(self.W)
self.gradb = np.zeros_like(self.b)
def updateOutput(self, input):
# Your code goes here. ################################################
self.output = np.add(np.dot(input, self.W.T), self.b)
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
self.gradInput = np.dot(gradOutput, self.W)
return self.gradInput
def accGradParameters(self, input, gradOutput):
# Your code goes here. ################################################
self.gradW = np.dot( gradOutput.T, input)
self.gradb = np.sum(gradOutput, axis=0)
def zeroGradParameters(self):
self.gradW.fill(0)
self.gradb.fill(0)
def getParameters(self):
return [self.W, self.b]
def getGradParameters(self):
return [self.gradW, self.gradb]
def __repr__(self):
s = self.W.shape
q = 'Linear %d -> %d' %(s[1],s[0])
return q
In [7]:
class SoftMax(Module):
def __init__(self):
super(SoftMax, self).__init__()
def updateOutput(self, input):
# start with normalization for numerical stability
self.output = np.subtract(input, input.max(axis=1, keepdims=True))
# Your code goes here. ################################################
self.output = np.exp(self.output)/np.sum(np.exp(self.output), axis=1, keepdims=True)
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
pred_grad = np.multiply(self.output, gradOutput)
self.gradInput = pred_grad - np.multiply(self.output, np.sum(pred_grad, axis=1, keepdims=True))
return self.gradInput
def __repr__(self):
return "SoftMax"
In [138]:
class LogSoftMax(Module):
def __init__(self):
super(LogSoftMax, self).__init__()
def updateOutput(self, input):
# start with normalization for numerical stability
self.output = np.subtract(input, input.max(axis=1, keepdims=True))
self.output = self.output - np.log(np.sum(np.exp(self.output), axis=1, keepdims=True))
# Your code goes here. ################################################
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
input_new_exp = np.exp(np.subtract(input, input.max(axis=1, keepdims=True)))
self.gradInput = gradOutput
self.gradInput -= np.multiply(input_new_exp/np.sum(input_new_exp, axis=1, keepdims=True),
np.sum(gradOutput, axis=1, keepdims=True))
return self.gradInput
def __repr__(self):
return "LogSoftMax"
One of the most significant recent ideas that impacted NNs a lot is Batch normalization. The idea is simple, yet effective: the features should be whitened ($mean = 0$, $std = 1$) all the way through NN. This improves the convergence for deep models letting it train them for days but not weeks. You are to implement the first part of the layer: features normalization. The second part (ChannelwiseScaling layer) is implemented below.
batch_size x n_featsbatch_size x n_featsThe layer should work as follows. While training (self.training == True) it transforms input as $$y = \frac{x - \mu} {\sqrt{\sigma + \epsilon}}$$
where $\mu$ and $\sigma$ - mean and variance of feature values in batch and $\epsilon$ is just a small number for numericall stability. Also during training, layer should maintain exponential moving average values for mean and variance:
self.moving_mean = self.moving_mean * alpha + batch_mean * (1 - alpha)
self.moving_variance = self.moving_variance * alpha + batch_variance * (1 - alpha)During testing (self.training == False) the layer normalizes input using moving_mean and moving_variance.
Note that decomposition of batch normalization on normalization itself and channelwise scaling here is just a common implementation choice. In general "batch normalization" always assumes normalization + scaling.
In [177]:
class BatchNormalization(Module):
EPS = 1e-3
def __init__(self, alpha = 0.):
super(BatchNormalization, self).__init__()
self.alpha = alpha
self.moving_mean = 0
self.moving_variance = 0
def updateOutput(self, input):
# Your code goes here. ################################################
# use self.EPS please
if (self.training == True):
self.mean = np.mean(input, axis=0, keepdims=True)
self.var = np.var(input, axis=0, keepdims=True)
self.moving_mean = self.moving_mean * self.alpha + self.mean * (1 - self.alpha)
self.moving_variance = self.moving_variance * self.alpha + self.var * (1 - self.alpha)
self.output = (input - self.mean)/np.sqrt(self.var + self.EPS)
else:
self.output = (input - self.moving_mean) / np.sqrt(self.moving_variance + self.EPS)
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
self.gradInput = input.shape[0] * gradOutput
self.gradInput -= np.sum(gradOutput, axis=0, keepdims=True)
self.gradInput -= self.output * np.sum(gradOutput * self.output, axis=0, keepdims=True)
self.gradInput = 1/(input.shape[0] * np.sqrt(self.var + self.EPS)) * self.gradInput
return self.gradInput
def __repr__(self):
return "BatchNormalization"
In [207]:
class ChannelwiseScaling(Module):
"""
Implements linear transform of input y = \gamma * x + \beta
where \gamma, \beta - learnable vectors of length x.shape[-1]
"""
def __init__(self, n_out):
super(ChannelwiseScaling, self).__init__()
stdv = 1./np.sqrt(n_out)
self.gamma = 1 #np.random.uniform(-stdv, stdv, size=n_out) # ACHTUNG!! 1
self.beta = 0 #0 #np.random.uniform(-stdv, stdv, size=n_out) # 0
self.gradGamma = np.zeros_like(self.gamma)
self.gradBeta = np.zeros_like(self.beta)
def updateOutput(self, input):
self.output = input * self.gamma + self.beta
return self.output
def updateGradInput(self, input, gradOutput):
self.gradInput = gradOutput * self.gamma
return self.gradInput
def accGradParameters(self, input, gradOutput):
self.gradBeta = np.sum(gradOutput, axis=0)
self.gradGamma = np.sum(gradOutput*input, axis=0)
def zeroGradParameters(self):
self.gradGamma.fill(0)
self.gradBeta.fill(0)
def getParameters(self):
return [self.gamma, self.beta]
def getGradParameters(self):
return [self.gradGamma, self.gradBeta]
def __repr__(self):
return "ChannelwiseScaling"
Practical notes. If BatchNormalization is placed after a linear transformation layer (including dense layer, convolutions, channelwise scaling) that implements function like y = weight * x + bias, than bias adding become useless and could be omitted since its effect will be discarded while batch mean subtraction. If BatchNormalization (followed by ChannelwiseScaling) is placed before a layer that propagates scale (including ReLU, LeakyReLU) followed by any linear transformation layer than parameter gamma in ChannelwiseScaling could be freezed since it could be absorbed into the linear transformation layer.
Implement dropout. The idea and implementation is really simple: just multimply the input by $Bernoulli(p)$ mask. Here $p$ is probability of an element to be zeroed.
This has proven to be an effective technique for regularization and preventing the co-adaptation of neurons.
While training (self.training == True) it should sample a mask on each iteration (for every batch), zero out elements and multiply elements by $1 / (1 - p)$. The latter is needed for keeping mean values of features close to mean values which will be in test mode. When testing this module should implement identity transform i.e. self.output = input.
batch_size x n_featsbatch_size x n_feats
In [215]:
class Dropout(Module):
def __init__(self, p=0.5):
super(Dropout, self).__init__()
self.p = p
self.mask = None
def updateOutput(self, input):
# Your code goes here. ################################################
if (self.training == True):
if (self.p == 0.0):
self.mask = np.ones_like(input)
else:
self.mask = np.random.binomial(1, 1 - self.p, input.shape)
self.output = np.multiply(input, self.mask) / (1 - self.p)
else:
self.output = input
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
self.gradInput = gradOutput * self.mask / (1 - self.p)
return self.gradInput
def __repr__(self):
return "Dropout"
Here's the complete example for the Rectified Linear Unit non-linearity (aka ReLU):
In [211]:
class ReLU(Module):
def __init__(self):
super(ReLU, self).__init__()
def updateOutput(self, input):
self.output = np.maximum(input, 0)
return self.output
def updateGradInput(self, input, gradOutput):
self.gradInput = np.multiply(gradOutput, input > 0)
return self.gradInput
def __repr__(self):
return "ReLU"
Implement Leaky Rectified Linear Unit. Expriment with slope.
In [210]:
class LeakyReLU(Module):
def __init__(self, slope = 0.03):
super(LeakyReLU, self).__init__()
self.slope = slope
def updateOutput(self, input):
# Your code goes here. ################################################
self.output = np.where(input <= 0, self.slope * input, input)
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
input_copy = np.where(input > 0, 1, input)
input_copy = np.where(input_copy < 0, self.slope, input_copy)
self.gradInput = np.multiply(gradOutput, input_copy)
return self.gradInput
def __repr__(self):
return "LeakyReLU"
Implement Exponential Linear Units activations.
In [214]:
class ELU(Module):
def __init__(self, alpha = 1.0):
super(ELU, self).__init__()
self.alpha = alpha
def updateOutput(self, input):
# Your code goes here. ################################################
self.output = np.where(input <= 0, self.alpha * (np.exp(input) - 1), input)
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
input_copy = np.where(input > 0, 1, input)
input_copy = np.where(input_copy <= 0, self.alpha * np.exp(input_copy), input_copy)
self.gradInput = np.multiply(input_copy, gradOutput)
return self.gradInput
def __repr__(self):
return "ELU"
Implement SoftPlus activations. Look, how they look a lot like ReLU.
In [68]:
class SoftPlus(Module):
def __init__(self):
super(SoftPlus, self).__init__()
def updateOutput(self, input):
# Your code goes here. ################################################
self.output = np.log(1 + np.exp(input))
return self.output
def updateGradInput(self, input, gradOutput):
# Your code goes here. ################################################
self.gradInput = np.exp(input)/(1 + np.exp(input)) * gradOutput
return self.gradInput
def __repr__(self):
return "SoftPlus"
Criterions are used to score the models answers.
In [1]:
class Criterion(object):
def __init__ (self):
self.output = None
self.gradInput = None
def forward(self, input, target):
"""
Given an input and a target, compute the loss function
associated to the criterion and return the result.
For consistency this function should not be overrided,
all the code goes in `updateOutput`.
"""
return self.updateOutput(input, target)
def backward(self, input, target):
"""
Given an input and a target, compute the gradients of the loss function
associated to the criterion and return the result.
For consistency this function should not be overrided,
all the code goes in `updateGradInput`.
"""
return self.updateGradInput(input, target)
def updateOutput(self, input, target):
"""
Function to override.
"""
return self.output
def updateGradInput(self, input, target):
"""
Function to override.
"""
return self.gradInput
def __repr__(self):
"""
Pretty printing. Should be overrided in every module if you want
to have readable description.
"""
return "Criterion"
The MSECriterion, which is basic L2 norm usually used for regression, is implemented here for you.
batch_size x n_featsbatch_size x n_feats
In [71]:
class MSECriterion(Criterion):
def __init__(self):
super(MSECriterion, self).__init__()
def updateOutput(self, input, target):
self.output = np.sum(np.power(input - target,2)) / input.shape[0]
return self.output
def updateGradInput(self, input, target):
self.gradInput = (input - target) * 2 / input.shape[0]
return self.gradInput
def __repr__(self):
return "MSECriterion"
You task is to implement the ClassNLLCriterion. It should implement multiclass log loss. Nevertheless there is a sum over y (target) in that formula,
remember that targets are one-hot encoded. This fact simplifies the computations a lot. Note, that criterions are the only places, where you divide by batch size. Also there is a small hack with adding small number to probabilities to avoid computing log(0).
batch_size x n_feats - probabilitiesbatch_size x n_feats - one-hot representation of ground truth
In [143]:
class ClassNLLCriterionUnstable(Criterion):
EPS = 1e-15
def __init__(self):
a = super(ClassNLLCriterionUnstable, self)
super(ClassNLLCriterionUnstable, self).__init__()
def updateOutput(self, input, target):
# Use this trick to avoid numerical errors
input_clamp = np.clip(input, self.EPS, 1 - self.EPS)
# Your code goes here. ################################################
self.output = - np.sum(np.multiply(target, np.log(input_clamp)))/input.shape[0]
return self.output
def updateGradInput(self, input, target):
# Use this trick to avoid numerical errors
input_clamp = np.clip(input, self.EPS, 1 - self.EPS)
# Your code goes here. ################################################
self.gradInput = - (target / input_clamp)/input.shape[0]
return self.gradInput
def __repr__(self):
return "ClassNLLCriterionUnstable"
batch_size x n_feats - log probabilitiesbatch_size x n_feats - one-hot representation of ground truthTask is similar to the previous one, but now the criterion input is the output of log-softmax layer. This decomposition allows us to avoid problems with computation of forward and backward of log().
In [6]:
class ClassNLLCriterion(Criterion):
def __init__(self):
a = super(ClassNLLCriterion, self)
super(ClassNLLCriterion, self).__init__()
def updateOutput(self, input, target):
# Your code goes here. ################################################
self.output = - np.sum(np.multiply(target, input))/input.shape[0]
return self.output
def updateGradInput(self, input, target):
# Your code goes here. ################################################
self.gradInput = - target/input.shape[0]
return self.gradInput
def __repr__(self):
return "ClassNLLCriterion"
batch_size x n_feats - embedding vectors (can be outputs of any layer, but usually the last one before the classification linear layer is used)labels - ground truth class indices (not one-hot ecodings)The contrastive loss pulls the examples of the same class closer (in terms of the embedding layer) and pushes the examples of different classes from each other.
This is the formula for the contrastive loss in the terms of embedding pairs: $$L_c(x_i, x_j, label_i, label_j) = \begin{cases}\frac{1}{2N_{pos}} \| x_i - x_j \|^{2}_{2} & label_i = label_j\\ \frac{1}{2N_{neg}} (\max(0, M - \| x_i - x_j \|_{2}))^2 & label_i \neq label_j \end{cases}$$ Where $M$ is a hyperparameter (constant), $N_{pos}$ and $N_{neg}$ - number of 'positive' (examples of the same class) and 'negative' pairs (examples of different classes).
You should generate all the possible pairs in the batch (ensure that your batch size is > 10, so that there are positive pairs), but remember to be efficient and use vectorize opetations, hint: https://medium.com/dataholiks-distillery/l2-distance-matrix-vectorization-trick-26aa3247ac6c. Then compute the Euclidean distances for them, and finally compute the loss value.
When computing the gradients with respect to inputs, you should (as always) use the chain rule: compute the loss gradients w.r.t distances, compute the distance gradients w.r.t input features (each example may take part in different pairs).
In [12]:
class ClassContrastiveCriterion(Criterion):
def __init__(self, M):
a = super(ClassContrastiveCriterion, self)
super(ClassContrastiveCriterion, self).__init__()
self.M = M #margin for punishing the negative pairs
def updateOutput(self, input, target):
# Your code goes here. ################################################
# Taken from the website given
def compute_distances_no_loops(X):
dists = -2 * np.dot(X, X.T) + np.sum(X**2, axis=1) + \
np.sum(X**2, axis=1)[:, np.newaxis]
return dists
# Calculate N_{neg} and N_{pos}
target_aug = (target[np.newaxis, :].T == target)
target_aug_neg = target_aug ^ np.eye(target.shape[0]).astype('bool')
Np = target_aug_neg.sum()
Nn = (target.shape[0] * (target.shape[0] - 1)) - Np
# Calculate distances
dist = compute_distances_no_loops(input)
loss = ~target_aug * 1/(2*Nn) * (np.maximum(0., self.M - np.sqrt(np.abs(dist)))**2)
loss = loss + target_aug_neg * dist / (2*Np)
self.output = loss.sum()
return self.output
def updateGradInput(self, input, target):
# Your code goes here. ################################################
def compute_distances_no_loops(X):
dists = X - X[:, np.newaxis]
norms = np.linalg.norm(dists, axis=-1)[:,:,np.newaxis]
norms = np.where(norms == 0, 1.0, norms)
ndist = dists / norms
return dists, norms, ndist
dist, norms, ndist = compute_distances_no_loops(input)
target_aug = target[np.newaxis, :]
target_aug = (target_aug.T == target)
target_aug_neg = target_aug ^ np.eye(target.shape[0]).astype('bool')
Np = target_aug_neg.sum()
Nn = (target.shape[0] * (target.shape[0] - 1)) - Np
loss_grad = - 2/Nn * ( ~target_aug[:,:,np.newaxis] * (np.maximum(0, self.M - norms) * ndist)).sum(axis=0)
loss_grad += 2/Np * (target_aug_neg[:,:,np.newaxis] * dist).sum(axis=0)
self.gradInput = loss_grad
return self.gradInput
def __repr__(self):
return "ClassContrastiveCriterion"
variables - list of lists of variables (one list per layer)gradients - list of lists of current gradients (same structure as for variables, one array for each var)config - dict with optimization parameters (learning_rate and momentum)state - dict with optimizator state (used to save accumulated gradients)
In [196]:
def sgd_momentum(variables, gradients, config, state):
# 'variables' and 'gradients' have complex structure, accumulated_grads will be stored in a simpler one
state.setdefault('accumulated_grads', {})
var_index = 0
for current_layer_vars, current_layer_grads in zip(variables, gradients):
for current_var, current_grad in zip(current_layer_vars, current_layer_grads):
old_grad = state['accumulated_grads'].setdefault(var_index, np.zeros_like(current_grad))
np.add(config['momentum'] * old_grad, config['learning_rate'] * current_grad, out=old_grad)
current_var -= old_grad
var_index += 1
variables - list of lists of variables (one list per layer)gradients - list of lists of current gradients (same structure as for variables, one array for each var)config - dict with optimization parameters (learning_rate, beta1, beta2, epsilon)state - dict with optimizator state (used to save 1st and 2nd moment for vars)Formulas for optimizer:
Current step learning rate: $$\text{lr}_t = \text{learning_rate} * \frac{\sqrt{1-\beta_2^t}} {1-\beta_1^t}$$ First moment of var: $$\mu_t = \beta_1 * \mu_{t-1} + (1 - \beta_1)*g$$ Second moment of var: $$v_t = \beta_2 * v_{t-1} + (1 - \beta_2)*g*g$$ New values of var: $$\text{variable} = \text{variable} - \text{lr}_t * \frac{\mu_t}{\sqrt{v_t} + \epsilon}$$
In [141]:
def adam_optimizer(variables, gradients, config, state):
# 'variables' and 'gradients' have complex structure, accumulated_grads will be stored in a simpler one
state.setdefault('m', {}) # first moment vars
state.setdefault('v', {}) # second moment vars
state.setdefault('t', 0) # timestamp
state['t'] += 1
for k in ['learning_rate', 'beta1', 'beta2', 'epsilon']:
assert k in config, config.keys()
var_index = 0
lr_t = config['learning_rate'] * np.sqrt(1 - config['beta2']**state['t']) / (1 - config['beta1']**state['t'])
for current_layer_vars, current_layer_grads in zip(variables, gradients):
for current_var, current_grad in zip(current_layer_vars, current_layer_grads):
var_first_moment = state['m'].setdefault(var_index, np.zeros_like(current_grad))
var_second_moment = state['v'].setdefault(var_index, np.zeros_like(current_grad))
# <YOUR CODE> #######################################
# update `current_var_first_moment`, `var_second_moment` and `current_var` values
np.add(config['beta1'] * var_first_moment, (1 - config['beta1']) * current_grad, out=var_first_moment)
np.add(config['beta2'] * var_second_moment, (1 - config['beta2']) * np.multiply(current_grad, current_grad), out=var_second_moment)
current_var -= lr_t * var_first_moment / (np.sqrt(var_second_moment) + config['epsilon'])
# small checks that you've updated the state; use np.add for rewriting np.arrays values
assert var_first_moment is state['m'].get(var_index)
assert var_second_moment is state['v'].get(var_index)
var_index += 1
In [35]:
class Flatten(Module):
def __init__(self):
super(Flatten, self).__init__()
def updateOutput(self, input):
self.output = input.reshape(len(input), -1)
return self.output
def updateGradInput(self, input, gradOutput):
self.gradInput = gradOutput.reshape(input.shape)
return self.gradInput
def __repr__(self):
return "Flatten"
In [ ]: