In [1]:

    

# Data: time-serie data from smartwatch or smartwatch data
# %matplotlib inline # for plt.show()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Data reading
# The smartwatch historical/time-seris data to visualize
# data_path = 'data/smartwatch_data/experimental_data_analysis/Basis_Watch_Data.csv'
# data_path = 'data/financial_data/USD_INR.csv'
data_path = 'data/bike_data/hour.csv'
data = pd.read_csv(data_path)

# Data: cleaning
# Getting rid of NaN
data = data.fillna(value=0.0)

# Showing the data file csv or comma separated value
data[:10]


    

    
Out[1]:







  

    

      

      
instant
      
dteday
      
season
      
yr
      
mnth
      
hr
      
holiday
      
weekday
      
workingday
      
weathersit
      
temp
      
atemp
      
hum
      
windspeed
      
casual
      
registered
      
cnt
    
  
  

    

      
0
      
1
      
2011-01-01
      
1
      
0
      
1
      
0
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.81
      
0.0000
      
3
      
13
      
16
    
    

      
1
      
2
      
2011-01-01
      
1
      
0
      
1
      
1
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
8
      
32
      
40
    
    

      
2
      
3
      
2011-01-01
      
1
      
0
      
1
      
2
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
5
      
27
      
32
    
    

      
3
      
4
      
2011-01-01
      
1
      
0
      
1
      
3
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
3
      
10
      
13
    
    

      
4
      
5
      
2011-01-01
      
1
      
0
      
1
      
4
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
0
      
1
      
1
    
    

      
5
      
6
      
2011-01-01
      
1
      
0
      
1
      
5
      
0
      
6
      
0
      
2
      
0.24
      
0.2576
      
0.75
      
0.0896
      
0
      
1
      
1
    
    

      
6
      
7
      
2011-01-01
      
1
      
0
      
1
      
6
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
2
      
0
      
2
    
    

      
7
      
8
      
2011-01-01
      
1
      
0
      
1
      
7
      
0
      
6
      
0
      
1
      
0.20
      
0.2576
      
0.86
      
0.0000
      
1
      
2
      
3
    
    

      
8
      
9
      
2011-01-01
      
1
      
0
      
1
      
8
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
1
      
7
      
8
    
    

      
9
      
10
      
2011-01-01
      
1
      
0
      
1
      
9
      
0
      
6
      
0
      
1
      
0.32
      
0.3485
      
0.76
      
0.0000
      
8
      
6
      
14
    
  

In [2]:

    

# # Plotting the smartwatch data before scaling/batch normalization
# data[:10000]['Price'].plot()
data[: 10].plot()
plt.legend()
plt.show()


    

In [3]:

    

data_array = np.array(data)
data_array.shape, data_array.dtype
data_main = np.array(data_array[:, -1:], dtype=float)
data_main.shape, data_main.dtype

plt.plot(data_main[:100])
plt.show()


    

In [4]:

    

mean = np.mean(data_main, axis=0)
std = np.std(data_main, axis=0)
std.shape, mean.shape, std.dtype, mean.dtype

data_norm = (data_main - mean) / std
plt.plot(data_norm[:100])
plt.show()
data_norm.mean(), data_norm.std(), data_norm.var(), data_norm.shape, data_norm.dtype


    

    













    
Out[4]:





(-1.0548364452851478e-16, 1.0, 1.0, (17379, 1), dtype('float64'))

In [5]:

    

train_data = data_norm[:16000] # the last dim/variable/feature
test_data = data_norm[16000:] # the last dim/variable/feature
train_data.shape, test_data.shape
X_train = train_data[0:15999]
Y_train = train_data[1:16000]
X_train.shape, Y_train.shape

plt.plot(X_train[:100])
plt.plot(Y_train[:100])
plt.show()


    

In [6]:

    

X_valid = test_data[0:1378] 
Y_valid = test_data[1:1379]
X_valid.shape, Y_valid.shape
plt.plot(X_valid[:100])
plt.plot(Y_valid[:100])
plt.show()


    

In [7]:

    

# Model or Network
import impl.layer as l
from impl.loss import *

class GRU:
    def __init__(self, D, H):
        self.D = D
        self.H = H
        self.losses = {'train':[], 'smooth train':[], 'valid': []}
        
        # Model params
        Z = H + D
        #         low, high = (-1.0), (+1.0)
        #         low, high = (-0.5), (+0.5)
        low_Z, high_Z = (-1. / np.sqrt(Z / 2.)), (1. / np.sqrt(Z / 2.))
        low_H, high_H = (-1. / np.sqrt(H / 2.)), (1. / np.sqrt(H / 2.))
        m = dict(
        #             Wz=np.random.randn(Z, H) / np.sqrt(Z / 2.),
        #             Wr=np.random.randn(Z, H) / np.sqrt(Z / 2.),
        #             Wh=np.random.randn(Z, H) / np.sqrt(Z / 2.),
        #             Wy=np.random.randn(H, D) / np.sqrt(H / 2.),
            Wz=np.random.uniform(size=(Z, H), low=low_Z, high=high_Z),
            Wr=np.random.uniform(size=(Z, H), low=low_Z, high=high_Z),
            Wh=np.random.uniform(size=(Z, H), low=low_Z, high=high_Z),
            Wy=np.random.uniform(size=(H, D), low=low_H, high=high_H),
            bz=np.zeros((1, H)),
            br=np.zeros((1, H)),
            bh=np.zeros((1, H)),
            by=np.zeros((1, D))
        )
        self.model = m
        
    def initial_state(self):
        return np.zeros((1, self.H))

    def forward(self, X, h, m):
        Wz, Wr, Wh, Wy = m['Wz'], m['Wr'], m['Wh'], m['Wy']
        bz, br, bh, by = m['bz'], m['br'], m['bh'], m['by']

        X_in = X.copy()
        h_in = h.copy()

        X = np.column_stack((h_in, X_in))

        hz, hz_cache = l.fc_forward(X, Wz, bz)
        hz, hz_sigm_cache = l.sigmoid_forward(hz)

        hr, hr_cache = l.fc_forward(X, Wr, br)
        hr, hr_sigm_cache = l.sigmoid_forward(hr)

        X = np.column_stack((hr * h_in, X_in))
        
        hh, hh_cache = l.fc_forward(X, Wh, bh)
        hh, hh_tanh_cache = l.tanh_forward(hh)

        # h = (1. - hz) * h_old + hz * hh
        # or
        h = ((1. - hz) * h_in) + (hz * hh)
        # or
        # h = h_in + hz (hh - h_in)

        y, y_cache = l.fc_forward(h, Wy, by)
        
        cache = (h_in, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache, hh, hh_cache, hh_tanh_cache, 
                 y_cache)

        return y, h, cache

    def backward(self, dy, dh, cache):
        h_in, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache, hh, hh_cache, hh_tanh_cache, y_cache = cache
        
        dh_out = dh.copy()

        dh, dWy, dby = l.fc_backward(dy, y_cache)
        dh += dh_out

        dh_in1 = (1. - hz) * dh
        dhh = hz * dh
        dhz = (hh * dh) - (h_in * dh)
        # or
        # dhz = (hh - h_in) * dh

        dhh = l.tanh_backward(dhh, hh_tanh_cache)
        dXh, dWh, dbh = l.fc_backward(dhh, hh_cache)

        dh = dXh[:, :self.H]
        dX_in2 = dXh[:, self.H:]
        dh_in2 = hr * dh

        dhr = h_in * dh
        dhr = l.sigmoid_backward(dhr, hr_sigm_cache)
        dXr, dWr, dbr = l.fc_backward(dhr, hr_cache)

        dhz = l.sigmoid_backward(dhz, hz_sigm_cache)
        dXz, dWz, dbz = l.fc_backward(dhz, hz_cache)

        dX = dXr + dXz
        dh_in3 = dX[:, :self.H]
        dX_in1 = dX[:, self.H:]

        dh = dh_in1 + dh_in2 + dh_in3
        dX = dX_in1 + dX_in2

        grad = dict(Wz=dWz, Wr=dWr, Wh=dWh, Wy=dWy, bz=dbz, br=dbr, bh=dbh, by=dby)
        
        return dX, dh, grad

    def train_forward(self, X_train, h):
        ys, fc_caches = [], []

        for X in X_train:
            X = X.reshape(1, -1) # X_1xn
            y, h, fc_cache = self.forward(X, h, self.model)
            fc_caches.append(fc_cache)
            ys.append(y)
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        caches = fc_caches
        
        return ys, caches
                                
    def loss_function(self, y_pred, y_train):
        loss, dys = 0.0, []

        for y, Y in zip(y_pred, y_train):
            loss += l2_regression(y_pred=y, y_train=Y)
            dy = dl2_regression(y_pred=y, y_train=Y)
            dys.append(dy)
            
        return loss, dys
    
    def train_backward(self, dys, caches):
        fc_caches = caches
        
        dh = np.zeros((1, self.H)) 
        grad = {key: np.zeros_like(val) for key, val in self.model.items()}
        grads= {key: np.zeros_like(val) for key, val in self.model.items()}

        for t in reversed(range(len(dys))):
            dy = dys[t].reshape(1, -1) # dy_1xn
            dX, dh, grad = self.backward(dy, dh, fc_caches[t])
            for key in grad.keys():
                grads[key] += grad[key]
                
        return dX, grads # TODO: dX is not used but this is a REMINDER that it exists!
    
    def test(self, X_seed, h, size):
        ys = []
        X = X_seed.reshape(1, -1)
        for _ in range(size):
            y, h, _ = self.forward(X, h, self.model)
            X = y.copy() # previous out for the next input for prediction
            ys.append(y) # list array
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        return ys


    

In [8]:

    

def get_minibatch(X, y, minibatch_size, shuffle):
    minibatches = []

    # for i in range(0, X.shape[0] - minibatch_size + 1, 1):
    for i in range(0, X.shape[0], minibatch_size):
        X_mini = X[i:(i + minibatch_size)]
        y_mini = y[i:(i + minibatch_size)]
        minibatches.append((X_mini, y_mini))

    return minibatches

def adam_rnn(nn, XY_train, XY_valid, alpha, mb_size, n_iter, print_after):
    X_train, y_train = XY_train
    X_valid, y_valid = XY_valid

    # Momentum
    M= {key: np.zeros_like(val) for key, val in nn.model.items()}
    R= {key: np.zeros_like(val) for key, val in nn.model.items()}
    
    # Learning decay: suggested by Justin Jhonson in Standford
    beta1 = .9
    beta2 = .99
    state = nn.initial_state()
    smooth_loss = 1.
    minibatches = get_minibatch(X_train, y_train, mb_size, shuffle=False)
    
    # Epochs: iterating through the whole data
    for iter in range(1, n_iter + 1):
        
        # Minibatches
        for idx in range(len(minibatches)):
            
            # Train the model
            X_mini, y_mini = minibatches[idx]
            ys, caches = nn.train_forward(X_mini, state)
            loss, dys = nn.loss_function(y_pred=ys, y_train=y_mini) #, alpha=alpha
            _, grads = nn.train_backward(dys, caches)
            nn.losses['train'].append(loss)
            smooth_loss = (0.999 * smooth_loss) + (0.001 * loss)
            nn.losses['smooth train'].append(smooth_loss)
            
            # Update the model
            for key in grads.keys(): #key, value: items
                M[key] = l.exp_running_avg(M[key], grads[key], beta1)
                R[key] = l.exp_running_avg(R[key], grads[key]**2, beta2)
                m_k_hat = M[key] / (1. - (beta1** iter))
                r_k_hat = R[key] / (1. - (beta2** iter))
                nn.model[key] -= alpha * m_k_hat / (np.sqrt(r_k_hat) + l.eps)

            # Validate the model (by testing)
            ys = nn.test(X_seed=X_valid[0], h=state, size=X_valid.shape[0]) # ys_tx1xn
            valid_loss, _ = nn.loss_function(y_pred=ys, y_train=Y_valid) #, alpha=alpha
            nn.losses['valid'].append(valid_loss)

        # Print the model loss/ error
        if iter % print_after == 0:
            print('Iter-{}, train loss: {:.8f}, valid loss: {:.8f}'.format(iter, loss, valid_loss))

    return nn


    

In [9]:

    

# Hyper-parameters
time_step = 128 # minibatch size: 32, 64, 128, or 256 Cache
n_iter = 300 # epochs
alpha = 1e-4 # learning_rate: 1e-3, 5e-4, 1e-4 - default choices
print_after = 1 # print training loss, valid, and test
num_hidden_units = 64 # num_hidden_units in hidden layer
num_input_units = X_train.shape[1] # X_txn: noise given by using all possible channels/ features

# Build the network and learning it or optimizing it using SGD
# def adam_rnn(nn, X_train, y_train, alpha=0.001, mb_size=256, n_iter=2000, print_after=100):
net = GRU(D=num_input_units, H=num_hidden_units) 

# Start learning using BP-SGD-ADAM
adam_rnn(nn=net, XY_train=(X_train, Y_train), XY_valid=(X_valid, Y_valid), alpha=alpha, mb_size=time_step,
         n_iter=n_iter, print_after=print_after)


    

    




Iter-1, train loss: 44.58209738, valid loss: 632.91933191
Iter-2, train loss: 40.54695846, valid loss: 632.66637837
Iter-3, train loss: 37.71799257, valid loss: 632.60491289
Iter-4, train loss: 35.58428305, valid loss: 632.76855749
Iter-5, train loss: 33.92057489, valid loss: 633.18523557
Iter-6, train loss: 32.53510503, valid loss: 633.86385130
Iter-7, train loss: 31.31833946, valid loss: 634.78073362
Iter-8, train loss: 30.21985034, valid loss: 635.89986633
Iter-9, train loss: 29.21195243, valid loss: 637.18510169
Iter-10, train loss: 28.27350289, valid loss: 638.62256895
Iter-11, train loss: 27.38539297, valid loss: 640.24333079
Iter-12, train loss: 26.52927704, valid loss: 642.14047857
Iter-13, train loss: 25.68583522, valid loss: 644.54115907
Iter-14, train loss: 24.83218451, valid loss: 647.99807266
Iter-15, train loss: 23.93790953, valid loss: 653.94559771
Iter-16, train loss: 22.96066335, valid loss: 667.20647982
Iter-17, train loss: 21.87208402, valid loss: 711.25293144
Iter-18, train loss: 20.79209626, valid loss: 833.67833469
Iter-19, train loss: 19.94571025, valid loss: 949.36800222
Iter-20, train loss: 19.17359472, valid loss: 1003.39326086
Iter-21, train loss: 18.42272812, valid loss: 1035.14147886
Iter-22, train loss: 17.70732478, valid loss: 1064.05691536
Iter-23, train loss: 17.02646726, valid loss: 1093.18573119
Iter-24, train loss: 16.38720287, valid loss: 1124.27358938
Iter-25, train loss: 15.79952002, valid loss: 1157.98638747
Iter-26, train loss: 15.26921459, valid loss: 1192.78980242
Iter-27, train loss: 14.79582800, valid loss: 1225.53373820
Iter-28, train loss: 14.37328138, valid loss: 1253.04899584
Iter-29, train loss: 13.99161565, valid loss: 1273.57927521
Iter-30, train loss: 13.63938377, valid loss: 1287.16406479
Iter-31, train loss: 13.30618957, valid loss: 1295.02569511
Iter-32, train loss: 12.98451247, valid loss: 1298.78298644
Iter-33, train loss: 12.67026385, valid loss: 1299.91913417
Iter-34, train loss: 12.36222902, valid loss: 1299.51749322
Iter-35, train loss: 12.06109509, valid loss: 1298.25241437
Iter-36, train loss: 11.76864224, valid loss: 1296.52828873
Iter-37, train loss: 11.48718188, valid loss: 1294.60497197
Iter-38, train loss: 11.21911583, valid loss: 1292.65173681
Iter-39, train loss: 10.96653990, valid loss: 1290.77796006
Iter-40, train loss: 10.73088248, valid loss: 1289.05051653
Iter-41, train loss: 10.51264052, valid loss: 1287.50312521
Iter-42, train loss: 10.31128986, valid loss: 1286.14697820
Iter-43, train loss: 10.12539955, valid loss: 1284.98349386
Iter-44, train loss: 9.95291721, valid loss: 1284.01474314
Iter-45, train loss: 9.79154347, valid loss: 1283.24949666
Iter-46, train loss: 9.63910021, valid loss: 1282.70576931
Iter-47, train loss: 9.49381840, valid loss: 1282.41095487
Iter-48, train loss: 9.35450428, valid loss: 1282.40012291
Iter-49, train loss: 9.22056990, valid loss: 1282.71296447
Iter-50, train loss: 9.09193386, valid loss: 1283.39008380
Iter-51, train loss: 8.96882218, valid loss: 1284.46931008
Iter-52, train loss: 8.85152738, valid loss: 1285.98219951
Iter-53, train loss: 8.74019789, valid loss: 1287.95024494
Iter-54, train loss: 8.63471045, valid loss: 1290.38001457
Iter-55, train loss: 8.53463450, valid loss: 1293.25722839
Iter-56, train loss: 8.43926135, valid loss: 1296.54286007
Iter-57, train loss: 8.34766314, valid loss: 1300.18105917
Iter-58, train loss: 8.25876002, valid loss: 1304.13861015
Iter-59, train loss: 8.17139007, valid loss: 1308.47737740
Iter-60, train loss: 8.08438162, valid loss: 1313.31334777
Iter-61, train loss: 7.99662155, valid loss: 1320.76469294
Iter-62, train loss: 7.90710544, valid loss: 1353.07048474
Iter-63, train loss: 7.81495534, valid loss: 1351.91100269
Iter-64, train loss: 7.71940051, valid loss: 1313.50216033
Iter-65, train loss: 7.61973056, valid loss: 1300.97002572
Iter-66, train loss: 7.51524142, valid loss: 1355.86786469
Iter-67, train loss: 7.40519710, valid loss: 1412.17208657
Iter-68, train loss: 7.28882677, valid loss: 1389.59434010
Iter-69, train loss: 7.16536736, valid loss: 1389.12811963
Iter-70, train loss: 7.03415158, valid loss: 1393.07286345
Iter-71, train loss: 6.89473201, valid loss: 1396.59963949
Iter-72, train loss: 6.74702933, valid loss: 1431.75669267
Iter-73, train loss: 6.59150542, valid loss: 1417.32272044
Iter-74, train loss: 6.42939851, valid loss: 1396.69828666
Iter-75, train loss: 6.26311335, valid loss: 1384.09865933
Iter-76, train loss: 6.09683922, valid loss: 1369.68774904
Iter-77, train loss: 5.93702280, valid loss: 1542.42844148
Iter-78, train loss: 5.79120675, valid loss: 1371.78323229
Iter-79, train loss: 5.66444238, valid loss: 1440.62141279
Iter-80, train loss: 5.55723957, valid loss: 1478.21024250
Iter-81, train loss: 5.46746325, valid loss: 1537.24596727
Iter-82, train loss: 5.39236120, valid loss: 1488.37688722
Iter-83, train loss: 5.32907921, valid loss: 1524.64487481
Iter-84, train loss: 5.27487874, valid loss: 1677.05151564
Iter-85, train loss: 5.22747777, valid loss: 1420.04400924
Iter-86, train loss: 5.18519441, valid loss: 1680.90228581
Iter-87, train loss: 5.14678409, valid loss: 1898.01717907
Iter-88, train loss: 5.11122856, valid loss: 1537.01503282
Iter-89, train loss: 5.07771027, valid loss: 1732.91261767
Iter-90, train loss: 5.04566987, valid loss: 1651.68073934
Iter-91, train loss: 5.01477285, valid loss: 1699.87281003
Iter-92, train loss: 4.98481248, valid loss: 1675.25393322
Iter-93, train loss: 4.95563462, valid loss: 1656.07195078
Iter-94, train loss: 4.92710465, valid loss: 1698.73018556
Iter-95, train loss: 4.89910059, valid loss: 1623.02812799
Iter-96, train loss: 4.87151646, valid loss: 1831.35457558
Iter-97, train loss: 4.84426739, valid loss: 1810.12817512
Iter-98, train loss: 4.81729328, valid loss: 1802.90046266
Iter-99, train loss: 4.79056012, valid loss: 1707.91447681
Iter-100, train loss: 4.76405920, valid loss: 1767.29786339
Iter-101, train loss: 4.73780470, valid loss: 1607.68958248
Iter-102, train loss: 4.71183007, valid loss: 1632.01865611
Iter-103, train loss: 4.68618390, valid loss: 1595.07711542
Iter-104, train loss: 4.66092534, valid loss: 1671.61630472
Iter-105, train loss: 4.63611953, valid loss: 1628.80220678
Iter-106, train loss: 4.61183324, valid loss: 1577.75312495
Iter-107, train loss: 4.58813079, valid loss: 1696.12878848
Iter-108, train loss: 4.56507051, valid loss: 1667.72066678
Iter-109, train loss: 4.54270177, valid loss: 1545.40401330
Iter-110, train loss: 4.52106266, valid loss: 1525.68568280
Iter-111, train loss: 4.50017837, valid loss: 1528.50820108
Iter-112, train loss: 4.48006021, valid loss: 1671.24213462
Iter-113, train loss: 4.46070535, valid loss: 1514.62565564
Iter-114, train loss: 4.44209712, valid loss: 1428.06478441
Iter-115, train loss: 4.42420590, valid loss: 1534.29263029
Iter-116, train loss: 4.40699045, valid loss: 1455.16168078
Iter-117, train loss: 4.39039975, valid loss: 1392.98763412
Iter-118, train loss: 4.37437509, valid loss: 1370.44203652
Iter-119, train loss: 4.35885258, valid loss: 1439.57795410
Iter-120, train loss: 4.34376566, valid loss: 1448.24171037
Iter-121, train loss: 4.32904786, valid loss: 1448.74313964
Iter-122, train loss: 4.31463532, valid loss: 1426.34792937
Iter-123, train loss: 4.30046923, valid loss: 1355.22762640
Iter-124, train loss: 4.28649782, valid loss: 1575.95723805
Iter-125, train loss: 4.27267788, valid loss: 1525.65606423
Iter-126, train loss: 4.25897563, valid loss: 1543.23752485
Iter-127, train loss: 4.24536702, valid loss: 1290.95858558
Iter-128, train loss: 4.23183729, valid loss: 1154.26916529
Iter-129, train loss: 4.21837995, valid loss: 1676.80553052
Iter-130, train loss: 4.20499537, valid loss: 1857.15959850
Iter-131, train loss: 4.19168895, valid loss: 1353.24434738
Iter-132, train loss: 4.17846930, valid loss: 1755.21249314
Iter-133, train loss: 4.16534643, valid loss: 1865.02579582
Iter-134, train loss: 4.15233023, valid loss: 1882.70011211
Iter-135, train loss: 4.13942929, valid loss: 1863.23980123
Iter-136, train loss: 4.12665018, valid loss: 1853.76237235
Iter-137, train loss: 4.11399729, valid loss: 1836.29949605
Iter-138, train loss: 4.10147291, valid loss: 1837.25391267
Iter-139, train loss: 4.08907784, valid loss: 1827.09134746
Iter-140, train loss: 4.07681216, valid loss: 1829.78591767
Iter-141, train loss: 4.06467608, valid loss: 1816.27877947
Iter-142, train loss: 4.05267080, valid loss: 1793.45753301
Iter-143, train loss: 4.04079927, valid loss: 1806.32020625
Iter-144, train loss: 4.02906671, valid loss: 1803.26602573
Iter-145, train loss: 4.01748092, valid loss: 1785.67921546
Iter-146, train loss: 4.00605222, valid loss: 1774.36709042
Iter-147, train loss: 3.99479327, valid loss: 1760.77270280
Iter-148, train loss: 3.98371856, valid loss: 1750.52649665
Iter-149, train loss: 3.97284379, valid loss: 1742.11806493
Iter-150, train loss: 3.96218513, valid loss: 1734.55040799
Iter-151, train loss: 3.95175844, valid loss: 1725.54831023
Iter-152, train loss: 3.94157850, valid loss: 1718.69118173
Iter-153, train loss: 3.93165835, valid loss: 1711.20385416
Iter-154, train loss: 3.92200871, valid loss: 1703.04252203
Iter-155, train loss: 3.91263754, valid loss: 1692.80286255
Iter-156, train loss: 3.90354975, valid loss: 1684.47686303
Iter-157, train loss: 3.89474711, valid loss: 1678.07430739
Iter-158, train loss: 3.88622818, valid loss: 1671.94146044
Iter-159, train loss: 3.87798853, valid loss: 1664.61655870
Iter-160, train loss: 3.87002087, valid loss: 1660.14394613
Iter-161, train loss: 3.86231547, valid loss: 1650.63026825
Iter-162, train loss: 3.85486044, valid loss: 1650.08670800
Iter-163, train loss: 3.84764217, valid loss: 1643.90615055
Iter-164, train loss: 3.84064572, valid loss: 1639.22328336
Iter-165, train loss: 3.83385524, valid loss: 1639.08673289
Iter-166, train loss: 3.82725432, valid loss: 1630.46977403
Iter-167, train loss: 3.82082638, valid loss: 1625.62420690
Iter-168, train loss: 3.81455494, valid loss: 1623.19555971
Iter-169, train loss: 3.80842390, valid loss: 1616.02907209
Iter-170, train loss: 3.80241778, valid loss: 1609.95681616
Iter-171, train loss: 3.79652186, valid loss: 1612.05634294
Iter-172, train loss: 3.79072234, valid loss: 1598.09847610
Iter-173, train loss: 3.78500644, valid loss: 1586.09615286
Iter-174, train loss: 3.77936241, valid loss: 1590.30858895
Iter-175, train loss: 3.77377960, valid loss: 1591.82632544
Iter-176, train loss: 3.76824843, valid loss: 1592.47105688
Iter-177, train loss: 3.76276039, valid loss: 1591.68425821
Iter-178, train loss: 3.75730795, valid loss: 1589.03569878
Iter-179, train loss: 3.75188454, valid loss: 1583.71650195
Iter-180, train loss: 3.74648446, valid loss: 1565.03501803
Iter-181, train loss: 3.74110280, valid loss: 1550.82939841
Iter-182, train loss: 3.73573537, valid loss: 1545.47807782
Iter-183, train loss: 3.73037860, valid loss: 1539.18877321
Iter-184, train loss: 3.72502945, valid loss: 1526.09309811
Iter-185, train loss: 3.71968536, valid loss: 1525.73307364
Iter-186, train loss: 3.71434412, valid loss: 1524.65256508
Iter-187, train loss: 3.70900385, valid loss: 1526.11151878
Iter-188, train loss: 3.70366288, valid loss: 1524.09682068
Iter-189, train loss: 3.69831975, valid loss: 1516.99494380
Iter-190, train loss: 3.69297306, valid loss: 1520.37485822
Iter-191, train loss: 3.68762153, valid loss: 1518.55884951
Iter-192, train loss: 3.68226388, valid loss: 1521.70542424
Iter-193, train loss: 3.67689880, valid loss: 1513.25594717
Iter-194, train loss: 3.67152495, valid loss: 1084.57896388
Iter-195, train loss: 3.66614094, valid loss: 950.27743757
Iter-196, train loss: 3.66074526, valid loss: 870.67807030
Iter-197, train loss: 3.65533631, valid loss: 778.43551016
Iter-198, train loss: 3.64991239, valid loss: 1020.37839647
Iter-199, train loss: 3.64447165, valid loss: 806.49897170
Iter-200, train loss: 3.63901217, valid loss: 915.85733372
Iter-201, train loss: 3.63353188, valid loss: 815.90165719
Iter-202, train loss: 3.62802862, valid loss: 1183.18626914
Iter-203, train loss: 3.62250013, valid loss: 1308.15796592
Iter-204, train loss: 3.61694407, valid loss: 1022.47263547
Iter-205, train loss: 3.61135803, valid loss: 1293.47471639
Iter-206, train loss: 3.60573956, valid loss: 1499.37794844
Iter-207, train loss: 3.60008617, valid loss: 1728.04160066
Iter-208, train loss: 3.59439537, valid loss: 1614.42247564
Iter-209, train loss: 3.58866471, valid loss: 1469.30367965
Iter-210, train loss: 3.58289178, valid loss: 1691.10845309
Iter-211, train loss: 3.57707423, valid loss: 1580.85791590
Iter-212, train loss: 3.57120986, valid loss: 1513.52250855
Iter-213, train loss: 3.56529658, valid loss: 1741.67340253
Iter-214, train loss: 3.55933249, valid loss: 1634.03056755
Iter-215, train loss: 3.55331589, valid loss: 1525.34650457
Iter-216, train loss: 3.54724532, valid loss: 1367.65423571
Iter-217, train loss: 3.54111958, valid loss: 1365.58059804
Iter-218, train loss: 3.53493779, valid loss: 1427.05570184
Iter-219, train loss: 3.52869937, valid loss: 1365.73471822
Iter-220, train loss: 3.52240412, valid loss: 1349.04257551
Iter-221, train loss: 3.51605219, valid loss: 1381.20332047
Iter-222, train loss: 3.50964414, valid loss: 1411.34121066
Iter-223, train loss: 3.50318093, valid loss: 1464.24905991
Iter-224, train loss: 3.49666396, valid loss: 1519.48908657
Iter-225, train loss: 3.49009504, valid loss: 1521.80241624
Iter-226, train loss: 3.48347641, valid loss: 1524.48173650
Iter-227, train loss: 3.47681077, valid loss: 1527.60532456
Iter-228, train loss: 3.47010120, valid loss: 1530.86706024
Iter-229, train loss: 3.46335122, valid loss: 1533.61091458
Iter-230, train loss: 3.45656475, valid loss: 1449.99110263
Iter-231, train loss: 3.44974605, valid loss: 1431.59077721
Iter-232, train loss: 3.44289976, valid loss: 1430.56353274
Iter-233, train loss: 3.43603083, valid loss: 1443.81056857
Iter-234, train loss: 3.42914449, valid loss: 1472.76771249
Iter-235, train loss: 3.42224624, valid loss: 1494.90676766
Iter-236, train loss: 3.41534177, valid loss: 1497.85472510
Iter-237, train loss: 3.40843694, valid loss: 1501.01096049
Iter-238, train loss: 3.40153777, valid loss: 1503.94644743
Iter-239, train loss: 3.39465034, valid loss: 1506.72649006
Iter-240, train loss: 3.38778078, valid loss: 1509.41323122
Iter-241, train loss: 3.38093524, valid loss: 1511.97989702
Iter-242, train loss: 3.37411982, valid loss: 1514.35712694
Iter-243, train loss: 3.36734052, valid loss: 1516.54590003
Iter-244, train loss: 3.36060328, valid loss: 1518.68722463
Iter-245, train loss: 3.35391383, valid loss: 1520.95950441
Iter-246, train loss: 3.34727775, valid loss: 1523.44565502
Iter-247, train loss: 3.34070041, valid loss: 1526.09800987
Iter-248, train loss: 3.33418694, valid loss: 1528.76397711
Iter-249, train loss: 3.32774220, valid loss: 1531.28425907
Iter-250, train loss: 3.32137077, valid loss: 1533.59664878
Iter-251, train loss: 3.31507695, valid loss: 1536.52938106
Iter-252, train loss: 3.30886474, valid loss: 1417.85036882
Iter-253, train loss: 3.30273781, valid loss: 1453.30368393
Iter-254, train loss: 3.29669953, valid loss: 1465.20984018
Iter-255, train loss: 3.29075292, valid loss: 1469.36616217
Iter-256, train loss: 3.28490073, valid loss: 1472.78360764
Iter-257, train loss: 3.27914536, valid loss: 1476.76615778
Iter-258, train loss: 3.27348891, valid loss: 1481.96904142
Iter-259, train loss: 3.26793319, valid loss: 1487.45688823
Iter-260, train loss: 3.26247971, valid loss: 1492.76473477
Iter-261, train loss: 3.25712972, valid loss: 1497.98719429
Iter-262, train loss: 3.25188419, valid loss: 1504.48001876
Iter-263, train loss: 3.24674384, valid loss: 1509.94431292
Iter-264, train loss: 3.24170917, valid loss: 1535.88130138
Iter-265, train loss: 3.23678045, valid loss: 1605.79972849
Iter-266, train loss: 3.23195773, valid loss: 1552.31450889
Iter-267, train loss: 3.22724091, valid loss: 1665.30772043
Iter-268, train loss: 3.22262967, valid loss: 1523.53485104
Iter-269, train loss: 3.21812356, valid loss: 1598.82530395
Iter-270, train loss: 3.21372199, valid loss: 1512.26379989
Iter-271, train loss: 3.20942423, valid loss: 1734.93906598
Iter-272, train loss: 3.20522942, valid loss: 1658.77161810
Iter-273, train loss: 3.20113663, valid loss: 1737.80648056
Iter-274, train loss: 3.19714482, valid loss: 1465.51328607
Iter-275, train loss: 3.19325289, valid loss: 1674.44796710
Iter-276, train loss: 3.18945966, valid loss: 1668.50041052
Iter-277, train loss: 3.18576389, valid loss: 1751.53211282
Iter-278, train loss: 3.18216432, valid loss: 1840.05180238
Iter-279, train loss: 3.17865964, valid loss: 1574.85195834
Iter-280, train loss: 3.17524852, valid loss: 1683.33417923
Iter-281, train loss: 3.17192960, valid loss: 1643.17741117
Iter-282, train loss: 3.16870153, valid loss: 1675.08927395
Iter-283, train loss: 3.16556292, valid loss: 1699.70593578
Iter-284, train loss: 3.16251243, valid loss: 1772.04454111
Iter-285, train loss: 3.15954868, valid loss: 1651.73242926
Iter-286, train loss: 3.15667034, valid loss: 1797.37088249
Iter-287, train loss: 3.15387605, valid loss: 1686.21173369
Iter-288, train loss: 3.15116451, valid loss: 1679.04205703
Iter-289, train loss: 3.14853442, valid loss: 1578.26046866
Iter-290, train loss: 3.14598450, valid loss: 1490.62861719
Iter-291, train loss: 3.14351350, valid loss: 1704.48620545
Iter-292, train loss: 3.14112018, valid loss: 1678.87757318
Iter-293, train loss: 3.13880336, valid loss: 1849.45831565
Iter-294, train loss: 3.13656185, valid loss: 1751.95335450
Iter-295, train loss: 3.13439449, valid loss: 1704.28162293
Iter-296, train loss: 3.13230016, valid loss: 1748.67934994
Iter-297, train loss: 3.13027775, valid loss: 1725.72571189
Iter-298, train loss: 3.12832617, valid loss: 1647.54264694
Iter-299, train loss: 3.12644435, valid loss: 1482.87983968
Iter-300, train loss: 3.12463124, valid loss: 1708.64356964






    
Out[9]:





<__main__.GRU at 0x7f44e9872390>

In [10]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['train'], label='Train loss')
plt.plot(net.losses['smooth train'], label='Train smooth loss')
plt.legend()
plt.show()


    

In [11]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['valid'], label='Validation loss')
plt.legend()
plt.show()


    

In [16]:

    

import matplotlib.pyplot as plt

y_pred = net.test(X_seed=X_valid[0], h=net.initial_state(), size=X_valid.shape[0]) # ys_tx1xn
y_pred.shape, Y_valid.shape

plt.plot(y_pred[100:350], label='y_pred')
plt.plot(Y_valid[100:350], label='Y_valid')
# plt.plot(X_valid[:100], label='X_valid')
plt.legend()
plt.show()


    

In [ ]:

    




    

In [ ]: