In [1]:

    

# Data: time-serie data from smartwatch or smartwatch data
# %matplotlib inline # for plt.show()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Data reading
# The smartwatch historical/time-seris data to visualize
# data_path = 'data/smartwatch_data/experimental_data_analysis/Basis_Watch_Data.csv'
# data_path = 'data/financial_data/USD_INR.csv'
data_path = 'data/bike_data/hour.csv'
data = pd.read_csv(data_path)

# Data: cleaning
# Getting rid of NaN
data = data.fillna(value=0.0)

# Showing the data file csv or comma separated value
data[:10]


    

    
Out[1]:







  

    

      

      
instant
      
dteday
      
season
      
yr
      
mnth
      
hr
      
holiday
      
weekday
      
workingday
      
weathersit
      
temp
      
atemp
      
hum
      
windspeed
      
casual
      
registered
      
cnt
    
  
  

    

      
0
      
1
      
2011-01-01
      
1
      
0
      
1
      
0
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.81
      
0.0000
      
3
      
13
      
16
    
    

      
1
      
2
      
2011-01-01
      
1
      
0
      
1
      
1
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
8
      
32
      
40
    
    

      
2
      
3
      
2011-01-01
      
1
      
0
      
1
      
2
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
5
      
27
      
32
    
    

      
3
      
4
      
2011-01-01
      
1
      
0
      
1
      
3
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
3
      
10
      
13
    
    

      
4
      
5
      
2011-01-01
      
1
      
0
      
1
      
4
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
0
      
1
      
1
    
    

      
5
      
6
      
2011-01-01
      
1
      
0
      
1
      
5
      
0
      
6
      
0
      
2
      
0.24
      
0.2576
      
0.75
      
0.0896
      
0
      
1
      
1
    
    

      
6
      
7
      
2011-01-01
      
1
      
0
      
1
      
6
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
2
      
0
      
2
    
    

      
7
      
8
      
2011-01-01
      
1
      
0
      
1
      
7
      
0
      
6
      
0
      
1
      
0.20
      
0.2576
      
0.86
      
0.0000
      
1
      
2
      
3
    
    

      
8
      
9
      
2011-01-01
      
1
      
0
      
1
      
8
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
1
      
7
      
8
    
    

      
9
      
10
      
2011-01-01
      
1
      
0
      
1
      
9
      
0
      
6
      
0
      
1
      
0.32
      
0.3485
      
0.76
      
0.0000
      
8
      
6
      
14
    
  

In [2]:

    

# # Plotting the smartwatch data before scaling/batch normalization
# data[:10000]['Price'].plot()
data[: 10].plot()
plt.legend()
plt.show()


    

In [3]:

    

data_array = np.array(data)
data_array.shape, data_array.dtype
data_main = np.array(data_array[:, -1:], dtype=float)
data_main.shape, data_main.dtype

plt.plot(data_main[:100])
plt.show()


    

In [4]:

    

mean = np.mean(data_main, axis=0)
std = np.std(data_main, axis=0)
std.shape, mean.shape, std.dtype, mean.dtype

data_norm = (data_main - mean) / std
plt.plot(data_norm[:100])
plt.show()
data_norm.mean(), data_norm.std(), data_norm.var(), data_norm.shape, data_norm.dtype


    

    













    
Out[4]:





(-1.0548364452851478e-16, 1.0, 1.0, (17379, 1), dtype('float64'))

In [5]:

    

train_data = data_norm[:16000] # the last dim/variable/feature
test_data = data_norm[16000:] # the last dim/variable/feature
train_data.shape, test_data.shape
X_train = train_data[0:15999]
Y_train = train_data[1:16000]
X_train.shape, Y_train.shape

plt.plot(X_train[:100])
plt.plot(Y_train[:100])
plt.show()


    

In [6]:

    

X_valid = test_data[0:1378] 
Y_valid = test_data[1:1379]
X_valid.shape, Y_valid.shape
plt.plot(X_valid[:100])
plt.plot(Y_valid[:100])
plt.show()


    

In [9]:

    

# Model or Network
import impl.layer as l
from impl.loss import *

class GRU:
    def __init__(self, D, H):
        self.D = D
        self.H = H
        self.losses = {'train':[], 'smooth train':[], 'valid': []}
        
        # Model params
        Z = H + D
#         low, high = (-1. / np.sqrt(Z / 2.)), (1. / np.sqrt(Z / 2.))
        low, high = (-1.0), (+1.0)
        m = dict(
#             Wz=np.random.randn(Z, H) / np.sqrt(Z / 2.),
#             Wr=np.random.randn(Z, H) / np.sqrt(Z / 2.),
#             Wh=np.random.randn(Z, H) / np.sqrt(Z / 2.),
#             Wy=np.random.randn(H, D) / np.sqrt(H / 2.),
            Wz=np.random.uniform(size=(Z, H), low=low, high=high),
            Wr=np.random.uniform(size=(Z, H), low=low, high=high),
            Wh=np.random.uniform(size=(Z, H), low=low, high=high),
            Wy=np.random.uniform(size=(H, D), low=low, high=high),
            bz=np.zeros((1, H)),
            br=np.zeros((1, H)),
            bh=np.zeros((1, H)),
            by=np.zeros((1, D))
        )
        self.model = m
        
    def initial_state(self):
        return np.zeros((1, self.H))

    def forward(self, X, h, m):
        Wz, Wr, Wh, Wy = m['Wz'], m['Wr'], m['Wh'], m['Wy']
        bz, br, bh, by = m['bz'], m['br'], m['bh'], m['by']

        X_in = X.copy()
        h_in = h.copy()

        X = np.column_stack((h_in, X_in))

        hz, hz_cache = l.fc_forward(X, Wz, bz)
        hz, hz_sigm_cache = l.sigmoid_forward(hz)

        hr, hr_cache = l.fc_forward(X, Wr, br)
        hr, hr_sigm_cache = l.sigmoid_forward(hr)

        X = np.column_stack((hr * h_in, X_in))
        
        hh, hh_cache = l.fc_forward(X, Wh, bh)
        hh, hh_tanh_cache = l.tanh_forward(hh)

        # h = (1. - hz) * h_old + hz * hh
        # or
        h = ((1. - hz) * h_in) + (hz * hh)
        # or
        # h = h_in + hz (hh - h_in)

        y, y_cache = l.fc_forward(h, Wy, by)
        
        cache = (h_in, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache, hh, hh_cache, hh_tanh_cache, 
                 y_cache)

        return y, h, cache

    def backward(self, dy, dh, cache):
        h_in, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache, hh, hh_cache, hh_tanh_cache, y_cache = cache
        
        dh_out = dh.copy()

        dh, dWy, dby = l.fc_backward(dy, y_cache)
        dh += dh_out

        dh_in1 = (1. - hz) * dh
        dhh = hz * dh
        dhz = (hh * dh) - (h_in * dh)
        # or
        # dhz = (hh - h_in) * dh

        dhh = l.tanh_backward(dhh, hh_tanh_cache)
        dXh, dWh, dbh = l.fc_backward(dhh, hh_cache)

        dh = dXh[:, :self.H]
        dX_in2 = dXh[:, self.H:]
        dh_in2 = hr * dh

        dhr = h_in * dh
        dhr = l.sigmoid_backward(dhr, hr_sigm_cache)
        dXr, dWr, dbr = l.fc_backward(dhr, hr_cache)

        dhz = l.sigmoid_backward(dhz, hz_sigm_cache)
        dXz, dWz, dbz = l.fc_backward(dhz, hz_cache)

        dX = dXr + dXz
        dh_in3 = dX[:, :self.H]
        dX_in1 = dX[:, self.H:]

        dh = dh_in1 + dh_in2 + dh_in3
        dX = dX_in1 + dX_in2

        grad = dict(Wz=dWz, Wr=dWr, Wh=dWh, Wy=dWy, bz=dbz, br=dbr, bh=dbh, by=dby)
        
        return dX, dh, grad

    def train_forward(self, X_train, h):
        ys, fc_caches = [], []

        for X in X_train:
            X = X.reshape(1, -1) # X_1xn
            y, h, fc_cache = self.forward(X, h, self.model)
            fc_caches.append(fc_cache)
            ys.append(y)
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        caches = fc_caches
        
        return ys, caches
                                
    def loss_function(self, y_pred, y_train):
        loss, dys = 0.0, []

        for y, Y in zip(y_pred, y_train):
            loss += l2_regression(y_pred=y, y_train=Y)
            dy = dl2_regression(y_pred=y, y_train=Y)
            dys.append(dy)
            
        return loss, dys
    
    def train_backward(self, dys, caches):
        fc_caches = caches
        
        dh = np.zeros((1, self.H)) 
        grad = {key: np.zeros_like(val) for key, val in self.model.items()}
        grads= {key: np.zeros_like(val) for key, val in self.model.items()}

        for t in reversed(range(len(dys))):
            dy = dys[t].reshape(1, -1) # dy_1xn
            dX, dh, grad = self.backward(dy, dh, fc_caches[t])
            for key in grad.keys():
                grads[key] += grad[key]
                
        return dX, grads # TODO: dX is not used but this is a REMINDER that it exists!
    
    def test(self, X_seed, h, size):
        ys = []
        X = X_seed.reshape(1, -1)
        for _ in range(size):
            y, h, _ = self.forward(X, h, self.model)
            X = y.copy() # previous out for the next input for prediction
            ys.append(y) # list array
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        return ys


    

In [10]:

    

def get_minibatch(X, y, minibatch_size, shuffle):
    minibatches = []

    # for i in range(0, X.shape[0] - minibatch_size + 1, 1):
    for i in range(0, X.shape[0], minibatch_size):
        X_mini = X[i:(i + minibatch_size)]
        y_mini = y[i:(i + minibatch_size)]
        minibatches.append((X_mini, y_mini))

    return minibatches

def adam_rnn(nn, XY_train, XY_valid, alpha, mb_size, n_iter, print_after):
    X_train, y_train = XY_train
    X_valid, y_valid = XY_valid

    # Momentum
    M= {key: np.zeros_like(val) for key, val in nn.model.items()}
    R= {key: np.zeros_like(val) for key, val in nn.model.items()}
    
    # Learning decay: suggested by Justin Jhonson in Standford
    beta1 = .9
    beta2 = .99
    state = nn.initial_state()
    smooth_loss = 1.
    minibatches = get_minibatch(X_train, y_train, mb_size, shuffle=False)
    
    # Epochs: iterating through the whole data
    for iter in range(1, n_iter + 1):
        
        # Minibatches
        for idx in range(len(minibatches)):
            
            # Train the model
            X_mini, y_mini = minibatches[idx]
            ys, caches = nn.train_forward(X_mini, state)
            loss, dys = nn.loss_function(y_pred=ys, y_train=y_mini) #, alpha=alpha
            _, grads = nn.train_backward(dys, caches)
            nn.losses['train'].append(loss)
            smooth_loss = (0.999 * smooth_loss) + (0.001 * loss)
            nn.losses['smooth train'].append(smooth_loss)
            
            # Update the model
            for key in grads.keys(): #key, value: items
                M[key] = l.exp_running_avg(M[key], grads[key], beta1)
                R[key] = l.exp_running_avg(R[key], grads[key]**2, beta2)
                m_k_hat = M[key] / (1. - (beta1** iter))
                r_k_hat = R[key] / (1. - (beta2** iter))
                nn.model[key] -= alpha * m_k_hat / (np.sqrt(r_k_hat) + l.eps)

            # Validate the model (by testing)
            ys = nn.test(X_seed=X_valid[0], h=state, size=X_valid.shape[0]) # ys_tx1xn
            valid_loss, _ = nn.loss_function(y_pred=ys, y_train=Y_valid) #, alpha=alpha
            nn.losses['valid'].append(valid_loss)

        # Print the model loss/ error
        if iter % print_after == 0:
            print('Iter-{}, train loss: {:.8f}, valid loss: {:.8f}'.format(iter, loss, valid_loss))

    return nn


    

In [11]:

    

# Hyper-parameters
time_step = 128 # minibatch size: 32, 64, 128, or 256 Cache
n_iter = 300 # epochs
alpha = 1e-4 # learning_rate: 1e-3, 5e-4, 1e-4 - default choices
print_after = 1 # print training loss, valid, and test
num_hidden_units = 64 # num_hidden_units in hidden layer
num_input_units = X_train.shape[1] # X_txn: noise given by using all possible channels/ features

# Build the network and learning it or optimizing it using SGD
# def adam_rnn(nn, X_train, y_train, alpha=0.001, mb_size=256, n_iter=2000, print_after=100):
net = GRU(D=num_input_units, H=num_hidden_units) 

# Start learning using BP-SGD-ADAM
adam_rnn(nn=net, XY_train=(X_train, Y_train), XY_valid=(X_valid, Y_valid), alpha=alpha, mb_size=time_step,
         n_iter=n_iter, print_after=print_after)


    

    




Iter-1, train loss: 1010.23920343, valid loss: 4126.05019475
Iter-2, train loss: 655.47122828, valid loss: 5240.29576610
Iter-3, train loss: 496.82786845, valid loss: 4024.98228671
Iter-4, train loss: 698.89187132, valid loss: 2950.45784678
Iter-5, train loss: 357.71172733, valid loss: 5632.88325312
Iter-6, train loss: 1577.55039824, valid loss: 1664.51193883
Iter-7, train loss: 293.12753071, valid loss: 3664.84253831
Iter-8, train loss: 588.63394319, valid loss: 3594.72061452
Iter-9, train loss: 594.57613241, valid loss: 4949.49675174
Iter-10, train loss: 930.99060964, valid loss: 4349.29336177
Iter-11, train loss: 1071.04782457, valid loss: 3331.48671411
Iter-12, train loss: 531.95746051, valid loss: 11601.24575665
Iter-13, train loss: 1050.19929969, valid loss: 6173.23826350
Iter-14, train loss: 618.67506610, valid loss: 3506.18111698
Iter-15, train loss: 844.61964754, valid loss: 3696.59627941
Iter-16, train loss: 542.23200049, valid loss: 5726.15221791
Iter-17, train loss: 752.29681062, valid loss: 3125.66756536
Iter-18, train loss: 429.51429342, valid loss: 3622.80872275
Iter-19, train loss: 580.94642741, valid loss: 3664.36451932
Iter-20, train loss: 595.96655323, valid loss: 2838.81640411
Iter-21, train loss: 1048.86217800, valid loss: 11853.54311306
Iter-22, train loss: 1674.97956908, valid loss: 2576.03287225
Iter-23, train loss: 1167.87002682, valid loss: 2511.05957454
Iter-24, train loss: 723.51502224, valid loss: 3594.14328261
Iter-25, train loss: 624.16609875, valid loss: 5214.09661196
Iter-26, train loss: 518.70292375, valid loss: 2777.22613189
Iter-27, train loss: 523.12226207, valid loss: 2584.43223553
Iter-28, train loss: 435.39801689, valid loss: 3439.44183907
Iter-29, train loss: 480.15502871, valid loss: 2885.05326588
Iter-30, train loss: 381.57689969, valid loss: 2880.72973849
Iter-31, train loss: 615.14977593, valid loss: 2923.84693284
Iter-32, train loss: 730.19110143, valid loss: 3623.97392949
Iter-33, train loss: 453.11005041, valid loss: 3460.62515641
Iter-34, train loss: 397.07313424, valid loss: 2734.68620805
Iter-35, train loss: 352.74380299, valid loss: 2378.20897828
Iter-36, train loss: 558.10406430, valid loss: 2287.27390525
Iter-37, train loss: 329.20105140, valid loss: 1486.33034250
Iter-38, train loss: 238.36990177, valid loss: 3542.56977110
Iter-39, train loss: 411.00850781, valid loss: 2894.71561118
Iter-40, train loss: 196.09689219, valid loss: 5511.43644011
Iter-41, train loss: 584.39342069, valid loss: 2402.00917219
Iter-42, train loss: 415.79822119, valid loss: 2632.29161228
Iter-43, train loss: 391.44540761, valid loss: 2705.56177385
Iter-44, train loss: 465.70320997, valid loss: 2247.66036214
Iter-45, train loss: 361.75608699, valid loss: 2952.71896535
Iter-46, train loss: 412.47399394, valid loss: 2457.02522972
Iter-47, train loss: 763.02290535, valid loss: 2485.21292092
Iter-48, train loss: 366.24687886, valid loss: 2358.06724728
Iter-49, train loss: 475.65508785, valid loss: 3085.55715714
Iter-50, train loss: 462.76812986, valid loss: 2920.69773021
Iter-51, train loss: 295.63846717, valid loss: 2531.58268033
Iter-52, train loss: 665.52145312, valid loss: 3078.24222626
Iter-53, train loss: 412.08915728, valid loss: 3003.70736247
Iter-54, train loss: 319.23901576, valid loss: 2376.10986360
Iter-55, train loss: 416.08068941, valid loss: 2167.39693347
Iter-56, train loss: 241.48030939, valid loss: 2272.06225792
Iter-57, train loss: 278.26476574, valid loss: 5122.52803676
Iter-58, train loss: 248.42946062, valid loss: 2943.93647661
Iter-59, train loss: 243.68001080, valid loss: 2097.59635249
Iter-60, train loss: 179.23403339, valid loss: 2255.76060729
Iter-61, train loss: 257.51722219, valid loss: 1862.72798106
Iter-62, train loss: 151.34106505, valid loss: 10876.91616127
Iter-63, train loss: 249.98746837, valid loss: 3246.37848971
Iter-64, train loss: 283.43042292, valid loss: 1851.51556002
Iter-65, train loss: 325.45756186, valid loss: 2085.61043520
Iter-66, train loss: 254.25191157, valid loss: 2627.46897643
Iter-67, train loss: 289.58635031, valid loss: 2717.23580230
Iter-68, train loss: 145.91909911, valid loss: 3116.78758665
Iter-69, train loss: 452.11113103, valid loss: 3334.71150771
Iter-70, train loss: 389.94079110, valid loss: 1853.21624246
Iter-71, train loss: 159.71240088, valid loss: 2803.86316388
Iter-72, train loss: 165.35644878, valid loss: 1874.38585252
Iter-73, train loss: 171.90418200, valid loss: 2568.22631061
Iter-74, train loss: 315.01975639, valid loss: 3040.39210477
Iter-75, train loss: 103.97179321, valid loss: 2820.16682639
Iter-76, train loss: 113.51649025, valid loss: 2638.44481904
Iter-77, train loss: 156.44039797, valid loss: 3776.06754391
Iter-78, train loss: 184.98635929, valid loss: 3578.34625111
Iter-79, train loss: 406.50325893, valid loss: 7291.31838307
Iter-80, train loss: 180.30071662, valid loss: 2605.77081720
Iter-81, train loss: 174.04860792, valid loss: 5341.97008607
Iter-82, train loss: 86.45091237, valid loss: 5471.33016878
Iter-83, train loss: 110.83195111, valid loss: 3198.69118829
Iter-84, train loss: 81.15801534, valid loss: 5082.70958344
Iter-85, train loss: 83.55202169, valid loss: 3547.90466978
Iter-86, train loss: 195.26426611, valid loss: 6220.57702265
Iter-87, train loss: 188.21850994, valid loss: 4845.28281321
Iter-88, train loss: 177.14022058, valid loss: 5828.70888212
Iter-89, train loss: 133.45794348, valid loss: 4105.76330987
Iter-90, train loss: 259.45008020, valid loss: 2592.55580020
Iter-91, train loss: 167.14841701, valid loss: 3996.15042218
Iter-92, train loss: 123.59366487, valid loss: 6521.50030356
Iter-93, train loss: 57.35682912, valid loss: 4807.05099988
Iter-94, train loss: 137.88282735, valid loss: 5561.43171928
Iter-95, train loss: 153.98271852, valid loss: 2478.26984355
Iter-96, train loss: 79.92328899, valid loss: 4528.10936986
Iter-97, train loss: 194.89394278, valid loss: 3540.59134153
Iter-98, train loss: 119.51816031, valid loss: 3461.14896698
Iter-99, train loss: 116.05794836, valid loss: 6323.36048171
Iter-100, train loss: 108.48164861, valid loss: 5170.56754130
Iter-101, train loss: 120.24408266, valid loss: 6186.37368843
Iter-102, train loss: 125.87016766, valid loss: 2946.13774737
Iter-103, train loss: 117.92968155, valid loss: 3698.58283450
Iter-104, train loss: 139.41027522, valid loss: 4912.67434791
Iter-105, train loss: 81.70672860, valid loss: 4439.53492712
Iter-106, train loss: 81.82609419, valid loss: 7335.85774795
Iter-107, train loss: 116.40727301, valid loss: 5237.03303831
Iter-108, train loss: 156.81267098, valid loss: 5292.29034489
Iter-109, train loss: 127.09528769, valid loss: 4389.76520387
Iter-110, train loss: 158.42934369, valid loss: 3280.22982491
Iter-111, train loss: 112.22484597, valid loss: 2829.03538560
Iter-112, train loss: 113.97850508, valid loss: 2841.18824785
Iter-113, train loss: 73.88791025, valid loss: 4982.93198603
Iter-114, train loss: 116.71009054, valid loss: 3744.89372858
Iter-115, train loss: 100.61554315, valid loss: 3829.03215050
Iter-116, train loss: 88.85280742, valid loss: 6704.94281497
Iter-117, train loss: 93.81902424, valid loss: 10987.50749114
Iter-118, train loss: 135.05548279, valid loss: 3371.35207074
Iter-119, train loss: 71.30298358, valid loss: 2325.10440461
Iter-120, train loss: 95.49935439, valid loss: 4386.76037801
Iter-121, train loss: 74.44098554, valid loss: 8031.70271118
Iter-122, train loss: 94.83167647, valid loss: 3794.25746166
Iter-123, train loss: 78.97371749, valid loss: 3943.30509656
Iter-124, train loss: 73.96375234, valid loss: 3395.80441621
Iter-125, train loss: 95.92564936, valid loss: 5149.72475018
Iter-126, train loss: 98.65678036, valid loss: 4203.77618840
Iter-127, train loss: 65.52949015, valid loss: 3969.59969268
Iter-128, train loss: 68.62031298, valid loss: 4141.53742398
Iter-129, train loss: 90.88173018, valid loss: 4032.03313967
Iter-130, train loss: 94.27610781, valid loss: 4407.74065114
Iter-131, train loss: 78.26154722, valid loss: 2329.91766365
Iter-132, train loss: 125.23531538, valid loss: 4246.84829728
Iter-133, train loss: 70.08726789, valid loss: 4252.37669142
Iter-134, train loss: 73.32719513, valid loss: 4621.50781391
Iter-135, train loss: 97.07420676, valid loss: 3321.55293600
Iter-136, train loss: 60.04033566, valid loss: 2953.11588939
Iter-137, train loss: 84.04971403, valid loss: 3281.58703819
Iter-138, train loss: 56.88272939, valid loss: 3086.67558047
Iter-139, train loss: 79.87336288, valid loss: 2949.84813544
Iter-140, train loss: 86.87127797, valid loss: 2705.56197778
Iter-141, train loss: 74.65771538, valid loss: 3175.66348708
Iter-142, train loss: 78.26756687, valid loss: 3104.95972448
Iter-143, train loss: 70.88136761, valid loss: 5106.17021875
Iter-144, train loss: 67.68271656, valid loss: 3431.88093184
Iter-145, train loss: 116.67625727, valid loss: 3498.84317423
Iter-146, train loss: 83.52491221, valid loss: 5168.60975712
Iter-147, train loss: 72.55364296, valid loss: 2573.75437647
Iter-148, train loss: 67.54851520, valid loss: 2057.54391039
Iter-149, train loss: 72.13970411, valid loss: 3146.02922727
Iter-150, train loss: 79.32525975, valid loss: 3538.09361071
Iter-151, train loss: 75.79079984, valid loss: 3257.31095167
Iter-152, train loss: 83.00849025, valid loss: 3773.61168818
Iter-153, train loss: 66.71898475, valid loss: 2535.47119453
Iter-154, train loss: 72.72237201, valid loss: 3354.74626417
Iter-155, train loss: 85.20875688, valid loss: 2488.91386906
Iter-156, train loss: 51.09966198, valid loss: 2655.40569836
Iter-157, train loss: 78.82678030, valid loss: 2752.97186717
Iter-158, train loss: 60.50845543, valid loss: 3355.04065785
Iter-159, train loss: 47.33465312, valid loss: 3191.59338112
Iter-160, train loss: 52.31229261, valid loss: 11606.30558307
Iter-161, train loss: 57.02308015, valid loss: 2742.97995380
Iter-162, train loss: 48.29616692, valid loss: 5032.37504672
Iter-163, train loss: 44.09658751, valid loss: 3357.09948903
Iter-164, train loss: 46.84474427, valid loss: 2270.31056357
Iter-165, train loss: 70.88574910, valid loss: 3354.82296873
Iter-166, train loss: 62.24416966, valid loss: 2988.33084260
Iter-167, train loss: 61.35878632, valid loss: 2950.02876478
Iter-168, train loss: 54.06710791, valid loss: 2971.76828401
Iter-169, train loss: 41.74236947, valid loss: 2785.21441412
Iter-170, train loss: 52.61461635, valid loss: 2083.42314587
Iter-171, train loss: 41.91957747, valid loss: 3525.95940782
Iter-172, train loss: 56.68228799, valid loss: 2858.97826579
Iter-173, train loss: 55.81631346, valid loss: 2535.22908489
Iter-174, train loss: 48.68408796, valid loss: 2482.81147439
Iter-175, train loss: 63.26496570, valid loss: 2626.76574053
Iter-176, train loss: 62.51962981, valid loss: 3910.44820097
Iter-177, train loss: 55.71782701, valid loss: 2855.86493619
Iter-178, train loss: 56.19097880, valid loss: 1793.36654931
Iter-179, train loss: 53.81969118, valid loss: 3388.05688236
Iter-180, train loss: 49.49626426, valid loss: 2382.14591906
Iter-181, train loss: 42.62818689, valid loss: 1921.98152681
Iter-182, train loss: 49.78242816, valid loss: 2462.95934015
Iter-183, train loss: 42.92480157, valid loss: 1990.86107673
Iter-184, train loss: 57.96294682, valid loss: 4730.73158047
Iter-185, train loss: 61.93503157, valid loss: 1836.15698902
Iter-186, train loss: 37.99107195, valid loss: 2017.55233921
Iter-187, train loss: 50.96707274, valid loss: 1522.70042638
Iter-188, train loss: 32.11623630, valid loss: 2223.99911960
Iter-189, train loss: 59.54214412, valid loss: 1697.90168351
Iter-190, train loss: 27.45481885, valid loss: 4280.46366256
Iter-191, train loss: 31.39557907, valid loss: 2808.35679734
Iter-192, train loss: 44.75207208, valid loss: 2033.45424885
Iter-193, train loss: 23.41708514, valid loss: 3452.67797114
Iter-194, train loss: 37.72263146, valid loss: 2221.70472901
Iter-195, train loss: 35.10935794, valid loss: 1840.89224984
Iter-196, train loss: 40.69619345, valid loss: 2645.16619864
Iter-197, train loss: 35.00621772, valid loss: 2252.76545716
Iter-198, train loss: 44.60468188, valid loss: 1875.96090592
Iter-199, train loss: 33.26809506, valid loss: 4409.60531146
Iter-200, train loss: 33.12003992, valid loss: 2188.68829123
Iter-201, train loss: 35.60397054, valid loss: 1379.57923699
Iter-202, train loss: 44.06986686, valid loss: 2457.20705007
Iter-203, train loss: 34.36228669, valid loss: 1771.70078891
Iter-204, train loss: 34.00849568, valid loss: 2894.05883060
Iter-205, train loss: 47.82246424, valid loss: 4238.42423731
Iter-206, train loss: 39.67811970, valid loss: 3311.10120231
Iter-207, train loss: 36.65144997, valid loss: 1713.58611009
Iter-208, train loss: 32.54802591, valid loss: 2344.94858227
Iter-209, train loss: 38.80901400, valid loss: 1248.60158620
Iter-210, train loss: 44.33940742, valid loss: 3299.91829375
Iter-211, train loss: 32.19617533, valid loss: 2506.36325700
Iter-212, train loss: 40.78845400, valid loss: 2003.16325748
Iter-213, train loss: 45.20607147, valid loss: 1479.18305739
Iter-214, train loss: 53.86550657, valid loss: 1443.58944365
Iter-215, train loss: 47.22658300, valid loss: 1327.11043708
Iter-216, train loss: 45.13307892, valid loss: 1323.34305781
Iter-217, train loss: 39.19245028, valid loss: 1239.19990367
Iter-218, train loss: 35.23592639, valid loss: 1311.10988987
Iter-219, train loss: 37.75854585, valid loss: 1706.76272733
Iter-220, train loss: 42.83961117, valid loss: 2298.94429120
Iter-221, train loss: 26.98698642, valid loss: 3584.46112943
Iter-222, train loss: 38.36961306, valid loss: 3017.50337262
Iter-223, train loss: 30.52007013, valid loss: 2433.43499924
Iter-224, train loss: 31.71190825, valid loss: 2012.86663940
Iter-225, train loss: 38.39125614, valid loss: 1401.77397267
Iter-226, train loss: 34.54662269, valid loss: 2757.51311835
Iter-227, train loss: 40.04210177, valid loss: 1549.66850589
Iter-228, train loss: 44.67128368, valid loss: 1602.09394460
Iter-229, train loss: 31.59809513, valid loss: 1792.75125576
Iter-230, train loss: 45.25429119, valid loss: 2147.41279786
Iter-231, train loss: 35.49438705, valid loss: 3802.29166562
Iter-232, train loss: 31.07219216, valid loss: 2197.06073178
Iter-233, train loss: 38.18338204, valid loss: 1449.37742125
Iter-234, train loss: 46.83786453, valid loss: 1858.68275656
Iter-235, train loss: 43.84822506, valid loss: 1299.08839037
Iter-236, train loss: 33.04158023, valid loss: 1275.67336269
Iter-237, train loss: 38.69819652, valid loss: 1229.52169875
Iter-238, train loss: 39.63292439, valid loss: 1614.36997231
Iter-239, train loss: 38.21218665, valid loss: 1903.38946101
Iter-240, train loss: 37.07494780, valid loss: 1201.33301529
Iter-241, train loss: 32.44126391, valid loss: 1767.60070418
Iter-242, train loss: 41.41690958, valid loss: 2520.71946808
Iter-243, train loss: 36.30860304, valid loss: 1094.33988720
Iter-244, train loss: 41.99067612, valid loss: 1667.30479209
Iter-245, train loss: 28.89361870, valid loss: 1325.60197322
Iter-246, train loss: 44.76802913, valid loss: 1165.02933554
Iter-247, train loss: 36.14137496, valid loss: 1366.95001367
Iter-248, train loss: 30.86283237, valid loss: 2413.19363603
Iter-249, train loss: 42.15661708, valid loss: 3103.93667401
Iter-250, train loss: 28.80407566, valid loss: 926.42404579
Iter-251, train loss: 35.46886404, valid loss: 1225.19364194
Iter-252, train loss: 27.26210865, valid loss: 2569.73048211
Iter-253, train loss: 30.39803349, valid loss: 1100.14734554
Iter-254, train loss: 35.80475268, valid loss: 2346.76774524
Iter-255, train loss: 26.86605650, valid loss: 1533.68550666
Iter-256, train loss: 33.40261386, valid loss: 1183.61355375
Iter-257, train loss: 31.02374560, valid loss: 4234.34891537
Iter-258, train loss: 31.05478536, valid loss: 1125.28518641
Iter-259, train loss: 35.35494180, valid loss: 3795.76299250
Iter-260, train loss: 38.08140065, valid loss: 995.90029841
Iter-261, train loss: 32.86183314, valid loss: 3476.09756330
Iter-262, train loss: 40.76818141, valid loss: 992.15218039
Iter-263, train loss: 33.81010085, valid loss: 930.45471959
Iter-264, train loss: 42.63088382, valid loss: 779.67525706
Iter-265, train loss: 42.97692710, valid loss: 1073.96869256
Iter-266, train loss: 29.74784618, valid loss: 969.43504813
Iter-267, train loss: 26.46824143, valid loss: 1402.84873029
Iter-268, train loss: 30.29274902, valid loss: 1184.09018057
Iter-269, train loss: 29.32599132, valid loss: 1297.50001915
Iter-270, train loss: 35.58758365, valid loss: 1438.20818952
Iter-271, train loss: 36.69447903, valid loss: 1066.87468733
Iter-272, train loss: 31.11534282, valid loss: 2219.87621734
Iter-273, train loss: 33.13175539, valid loss: 1141.48435970
Iter-274, train loss: 30.81763187, valid loss: 1033.74098681
Iter-275, train loss: 39.33081706, valid loss: 819.26362193
Iter-276, train loss: 31.28272169, valid loss: 2635.61209436
Iter-277, train loss: 35.66576841, valid loss: 886.17319637
Iter-278, train loss: 32.55290413, valid loss: 883.04796711
Iter-279, train loss: 35.56412516, valid loss: 1136.30951525
Iter-280, train loss: 34.30937276, valid loss: 1030.73127007
Iter-281, train loss: 30.98953545, valid loss: 1343.82710829
Iter-282, train loss: 38.94838419, valid loss: 1196.54549148
Iter-283, train loss: 35.37720882, valid loss: 700.83037956
Iter-284, train loss: 34.73281943, valid loss: 1098.88854522
Iter-285, train loss: 42.71361409, valid loss: 1176.71778739
Iter-286, train loss: 34.25994456, valid loss: 2863.02758392
Iter-287, train loss: 36.91947265, valid loss: 920.33666722
Iter-288, train loss: 27.13263554, valid loss: 1029.12125211
Iter-289, train loss: 38.59340423, valid loss: 1010.20216031
Iter-290, train loss: 31.95075470, valid loss: 2036.99052818
Iter-291, train loss: 35.75600238, valid loss: 2412.85758902
Iter-292, train loss: 37.85099030, valid loss: 1198.89284175
Iter-293, train loss: 27.51519476, valid loss: 1103.63753139
Iter-294, train loss: 32.02869833, valid loss: 2517.50911940
Iter-295, train loss: 29.53163464, valid loss: 1105.16325794
Iter-296, train loss: 36.38525774, valid loss: 1133.55885351
Iter-297, train loss: 34.06685274, valid loss: 1316.75205932
Iter-298, train loss: 36.02527976, valid loss: 1224.54141078
Iter-299, train loss: 33.75144931, valid loss: 1011.72403966
Iter-300, train loss: 36.60589088, valid loss: 894.54065864






    
Out[11]:





<__main__.GRU at 0x7fbefef4bf28>

In [12]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['train'], label='Train loss')
plt.plot(net.losses['smooth train'], label='Train smooth loss')
plt.legend()
plt.show()


    

In [13]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['valid'], label='Validation loss')
plt.legend()
plt.show()


    

In [18]:

    

import matplotlib.pyplot as plt

y_pred = net.test(X_seed=X_valid[0], h=net.initial_state(), size=X_valid.shape[0]) # ys_tx1xn
y_pred.shape, Y_valid.shape

plt.plot(y_pred[:], label='y_pred')
plt.plot(Y_valid[:], label='Y_valid')
# plt.plot(X_valid[:100], label='X_valid')
plt.legend()
plt.show()


    

In [ ]:

    




    

In [ ]: