In [1]:

    

# Data: time-serie data from smartwatch or smartwatch data
# %matplotlib inline # for plt.show()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Data reading
# The smartwatch historical/time-seris data to visualize
# data_path = 'data/smartwatch_data/experimental_data_analysis/Basis_Watch_Data.csv'
# data_path = 'data/financial_data/USD_INR.csv'
data_path = 'data/bike_data/hour.csv'
data = pd.read_csv(data_path)

# Data: cleaning
# Getting rid of NaN
data = data.fillna(value=0.0)

# Showing the data file csv or comma separated value
data[:10]


    

    
Out[1]:







  

    

      

      
instant
      
dteday
      
season
      
yr
      
mnth
      
hr
      
holiday
      
weekday
      
workingday
      
weathersit
      
temp
      
atemp
      
hum
      
windspeed
      
casual
      
registered
      
cnt
    
  
  

    

      
0
      
1
      
2011-01-01
      
1
      
0
      
1
      
0
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.81
      
0.0000
      
3
      
13
      
16
    
    

      
1
      
2
      
2011-01-01
      
1
      
0
      
1
      
1
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
8
      
32
      
40
    
    

      
2
      
3
      
2011-01-01
      
1
      
0
      
1
      
2
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
5
      
27
      
32
    
    

      
3
      
4
      
2011-01-01
      
1
      
0
      
1
      
3
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
3
      
10
      
13
    
    

      
4
      
5
      
2011-01-01
      
1
      
0
      
1
      
4
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
0
      
1
      
1
    
    

      
5
      
6
      
2011-01-01
      
1
      
0
      
1
      
5
      
0
      
6
      
0
      
2
      
0.24
      
0.2576
      
0.75
      
0.0896
      
0
      
1
      
1
    
    

      
6
      
7
      
2011-01-01
      
1
      
0
      
1
      
6
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
2
      
0
      
2
    
    

      
7
      
8
      
2011-01-01
      
1
      
0
      
1
      
7
      
0
      
6
      
0
      
1
      
0.20
      
0.2576
      
0.86
      
0.0000
      
1
      
2
      
3
    
    

      
8
      
9
      
2011-01-01
      
1
      
0
      
1
      
8
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
1
      
7
      
8
    
    

      
9
      
10
      
2011-01-01
      
1
      
0
      
1
      
9
      
0
      
6
      
0
      
1
      
0.32
      
0.3485
      
0.76
      
0.0000
      
8
      
6
      
14
    
  

In [2]:

    

# # Plotting the smartwatch data before scaling/batch normalization
# data[:10000]['Price'].plot()
data[: 10].plot()
plt.legend()
plt.show()


    

In [3]:

    

data_array = np.array(data)
data_array.shape, data_array.dtype
data_main = np.array(data_array[:, -1:], dtype=float)
data_main.shape, data_main.dtype

plt.plot(data_main[:100])
plt.show()


    

In [4]:

    

mean = np.mean(data_main, axis=0)
std = np.std(data_main, axis=0)
std.shape, mean.shape, std.dtype, mean.dtype

data_norm = (data_main - mean) / std
plt.plot(data_norm[:100])
plt.show()
data_norm.mean(), data_norm.std(), data_norm.var(), data_norm.shape, data_norm.dtype


    

    













    
Out[4]:





(-1.0548364452851478e-16, 1.0, 1.0, (17379, 1), dtype('float64'))

In [5]:

    

train_data = data_norm[:16000] # the last dim/variable/feature
test_data = data_norm[16000:] # the last dim/variable/feature
train_data.shape, test_data.shape
X_train = train_data[0:15999]
Y_train = train_data[1:16000]
X_train.shape, Y_train.shape

plt.plot(X_train[:100])
plt.plot(Y_train[:100])
plt.show()


    

In [6]:

    

X_valid = test_data[0:1378] 
Y_valid = test_data[1:1379]
X_valid.shape, Y_valid.shape
plt.plot(X_valid[:100])
plt.plot(Y_valid[:100])
plt.show()


    

In [43]:

    

# Model or Network
import impl.layer as l
from impl.loss import *

class GRU:
    def __init__(self, D, H, p_dropout):
        self.D = D
        self.H = H
        self.p_dropout = p_dropout
        self.losses = {'train':[], 'smooth train':[], 'valid': []}
        
        # Model params
        Z = H + D
        m = dict(
            Wz=np.random.randn(Z, H) / np.sqrt(Z / 2.),
            Wr=np.random.randn(Z, H) / np.sqrt(Z / 2.),
            Wh=np.random.randn(Z, H) / np.sqrt(Z / 2.),
            Wy=np.random.randn(H, D) / np.sqrt(H / 2.),
            bz=np.zeros((1, H)),
            br=np.zeros((1, H)),
            bh=np.zeros((1, H)),
            by=np.zeros((1, D))
        )
        self.model = m
        
    def initial_state(self):
        return np.zeros((1, self.H))

    def forward(self, X, h, m):
        Wz, Wr, Wh, Wy = m['Wz'], m['Wr'], m['Wh'], m['Wy']
        bz, br, bh, by = m['bz'], m['br'], m['bh'], m['by']

        X_in = X.copy()
        h_in = h.copy()

        X = np.column_stack((h_in, X_in))

        hz, hz_cache = l.fc_forward(X, Wz, bz)
        hz, hz_sigm_cache = l.sigmoid_forward(hz)

        hr, hr_cache = l.fc_forward(X, Wr, br)
        hr, hr_sigm_cache = l.sigmoid_forward(hr)

        X = np.column_stack((hr * h_in, X_in))
        
        hh, hh_cache = l.fc_forward(X, Wh, bh)
        hh, hh_tanh_cache = l.tanh_forward(hh)

        # h = (1. - hz) * h_old + hz * hh
        # or
        h = ((1. - hz) * h_in) + (hz * hh)
        # or
        # h = h_in + hz (hh - h_in)

        y, y_cache = l.fc_forward(h, Wy, by)
        
        cache = (h_in, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache, hh, hh_cache, hh_tanh_cache, 
                 y_cache)

        return y, h, cache

    def backward(self, dy, dh, cache):
        h_in, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache, hh, hh_cache, hh_tanh_cache, y_cache = cache
        
        dh_out = dh.copy()

        dh, dWy, dby = l.fc_backward(dy, y_cache)
        dh += dh_out

        dh_in1 = (1. - hz) * dh
        dhh = hz * dh
        dhz = (hh * dh) - (h_in * dh)
        # or
        # dhz = (hh - h_in) * dh

        dhh = l.tanh_backward(dhh, hh_tanh_cache)
        dXh, dWh, dbh = l.fc_backward(dhh, hh_cache)

        dh = dXh[:, :self.H]
        dX_in2 = dXh[:, self.H:]
        dh_in2 = hr * dh

        dhr = h_in * dh
        dhr = l.sigmoid_backward(dhr, hr_sigm_cache)
        dXr, dWr, dbr = l.fc_backward(dhr, hr_cache)

        dhz = l.sigmoid_backward(dhz, hz_sigm_cache)
        dXz, dWz, dbz = l.fc_backward(dhz, hz_cache)

        dX = dXr + dXz
        dh_in3 = dX[:, :self.H]
        dX_in1 = dX[:, self.H:]

        dh = dh_in1 + dh_in2 + dh_in3
        dX = dX_in1 + dX_in2

        grad = dict(Wz=dWz, Wr=dWr, Wh=dWh, Wy=dWy, bz=dbz, br=dbr, bh=dbh, by=dby)
        
        return dX, dh, grad

    def train_forward(self, X_train, h):
        ys, fc_caches, do_caches = [], [], []

        for X in X_train:
            X = X.reshape(1, -1) # X_1xn
            y, h, fc_cache = self.forward(X, h, self.model)
            y, do_cache = l.dropout_forward(y, self.p_dropout)
            fc_caches.append(fc_cache)
            do_caches.append(do_cache)
            ys.append(y)
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        caches = (fc_caches, do_caches)
        
        return ys, caches
                                
    def loss_function(self, y_pred, y_train):
        loss, dys = 0.0, []

        for y, Y in zip(y_pred, y_train):
            loss += l2_regression(y_pred=y, y_train=Y)
            dy = dl2_regression(y_pred=y, y_train=Y)
            dys.append(dy)
            
        return loss, dys
    
    def train_backward(self, dys, caches):
        fc_caches, do_caches = caches
        
        dh = np.zeros((1, self.H)) 
        grad = {key: np.zeros_like(val) for key, val in self.model.items()}
        grads= {key: np.zeros_like(val) for key, val in self.model.items()}

        for t in reversed(range(len(dys))):
            dy = dys[t].reshape(1, -1) # dy_1xn
            dy = l.dropout_backward(dy, do_caches[t])
            dX, dh, grad = self.backward(dy, dh, fc_caches[t])
            for key in grad.keys():
                grads[key] += grad[key]
                
        return dX, grads # TODO: dX is not used but this is a REMINDER that it exists!
    
    def test(self, X_seed, h, size):
        ys = []
        X = X_seed.reshape(1, -1)
        for _ in range(size):
            y, h, _ = self.forward(X, h, self.model)
            X = y.copy() # previous out for the next input for prediction
            ys.append(y) # list array
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        return ys


    

In [44]:

    

def get_minibatch(X, y, minibatch_size, shuffle):
    minibatches = []

    # for i in range(0, X.shape[0] - minibatch_size + 1, 1):
    for i in range(0, X.shape[0], minibatch_size):
        X_mini = X[i:(i + minibatch_size)]
        y_mini = y[i:(i + minibatch_size)]
        minibatches.append((X_mini, y_mini))

    return minibatches

def adam_rnn(nn, XY_train, XY_valid, alpha, mb_size, n_iter, print_after):
    X_train, y_train = XY_train
    X_valid, y_valid = XY_valid

    # Momentum
    M= {key: np.zeros_like(val) for key, val in nn.model.items()}
    R= {key: np.zeros_like(val) for key, val in nn.model.items()}
    
    # Learning decay: suggested by Justin Jhonson in Standford
    beta1 = .9
    beta2 = .99
    state = nn.initial_state()
    smooth_loss = 1.
    minibatches = get_minibatch(X_train, y_train, mb_size, shuffle=False)
    
    # Epochs: iterating through the whole data
    for iter in range(1, n_iter + 1):
        
        # Minibatches
        for idx in range(len(minibatches)):
            
            # Train the model
            X_mini, y_mini = minibatches[idx]
            ys, caches = nn.train_forward(X_mini, state)
            loss, dys = nn.loss_function(y_pred=ys, y_train=y_mini) #, alpha=alpha
            _, grads = nn.train_backward(dys, caches)
            nn.losses['train'].append(loss)
            smooth_loss = (0.999 * smooth_loss) + (0.001 * loss)
            nn.losses['smooth train'].append(smooth_loss)
            
            # Update the model
            for key in grads.keys(): #key, value: items
                M[key] = l.exp_running_avg(M[key], grads[key], beta1)
                R[key] = l.exp_running_avg(R[key], grads[key]**2, beta2)
                m_k_hat = M[key] / (1. - (beta1** iter))
                r_k_hat = R[key] / (1. - (beta2** iter))
                nn.model[key] -= alpha * m_k_hat / (np.sqrt(r_k_hat) + l.eps)

            # Validate the model (by testing)
            ys = nn.test(X_seed=X_valid[0], h=state, size=X_valid.shape[0]) # ys_tx1xn
            valid_loss, _ = nn.loss_function(y_pred=ys, y_train=Y_valid) #, alpha=alpha
            nn.losses['valid'].append(valid_loss)

        # Print the model loss/ error
        if iter % print_after == 0:
            print('Iter-{}, train loss: {:.8f}, valid loss: {:.8f}'.format(iter, loss, valid_loss))

    return nn


    

In [ ]:

    

# Hyper-parameters
time_step = 64 # minibatch size: 32, 64, 128, or 256 Cache
n_iter = 1000 # epochs
alpha = 1e-4 # learning_rate: 1e-3, 5e-4, 1e-4 - default choices
print_after = 1 # print training loss, valid, and test
num_hidden_units = 64 # num_hidden_units in hidden layer
num_input_units = X_train.shape[1] # X_txn: noise given by using all possible channels/ features
keep_prob = 0.95 # p_dropout == keep_prob: keeping neurons/units function - default 0.95 to 0.9 based on SELU-Dropout

# Build the network and learning it or optimizing it using SGD
# def adam_rnn(nn, X_train, y_train, alpha=0.001, mb_size=256, n_iter=2000, print_after=100):
net = GRU(D=num_input_units, H=num_hidden_units, p_dropout=keep_prob) # , lam=lam, L=num_layers, p_dropout=p_dropout

# Start learning using BP-SGD-ADAM
adam_rnn(nn=net, XY_train=(X_train, Y_train), XY_valid=(X_valid, Y_valid), alpha=alpha, mb_size=time_step,
         n_iter=n_iter, print_after=print_after)


    

    




Iter-1, train loss: 29.35954020, valid loss: 633.58082002
Iter-2, train loss: 29.24590501, valid loss: 637.00816697
Iter-3, train loss: 29.36778551, valid loss: 644.41389150
Iter-4, train loss: 30.02981882, valid loss: 653.71110215
Iter-5, train loss: 29.90908806, valid loss: 675.14409326
Iter-6, train loss: 29.89713651, valid loss: 729.88832539
Iter-7, train loss: 27.77186928, valid loss: 771.37486824
Iter-8, train loss: 29.79618059, valid loss: 772.94180518
Iter-9, train loss: 29.27332902, valid loss: 841.80226888
Iter-10, train loss: 30.10320203, valid loss: 1100.35194537
Iter-11, train loss: 30.44829880, valid loss: 941.26471067
Iter-12, train loss: 24.50909613, valid loss: 1232.67484074
Iter-13, train loss: 28.62266071, valid loss: 1320.03376802
Iter-14, train loss: 28.99165305, valid loss: 1227.12783257
Iter-15, train loss: 29.96870139, valid loss: 1399.65997442
Iter-16, train loss: 29.83021116, valid loss: 1281.16821810
Iter-17, train loss: 30.30306196, valid loss: 1247.05113255
Iter-18, train loss: 30.56970278, valid loss: 1359.70060038
Iter-19, train loss: 29.04171333, valid loss: 1383.30598119
Iter-20, train loss: 30.31434895, valid loss: 1401.25843703
Iter-21, train loss: 30.63805453, valid loss: 1692.13554222
Iter-22, train loss: 30.78931562, valid loss: 1363.15608867
Iter-23, train loss: 27.56631158, valid loss: 1500.17810185
Iter-24, train loss: 30.30449860, valid loss: 1488.84450136
Iter-25, train loss: 30.24169797, valid loss: 1525.45430277
Iter-26, train loss: 29.66620414, valid loss: 1599.95413118
Iter-27, train loss: 30.69297150, valid loss: 1665.72259160
Iter-28, train loss: 29.42796542, valid loss: 1412.05998255
Iter-29, train loss: 30.94556456, valid loss: 1535.44063774
Iter-30, train loss: 27.23993875, valid loss: 1563.12361283
Iter-31, train loss: 30.41169787, valid loss: 1628.58923699
Iter-32, train loss: 30.02718907, valid loss: 1558.14795944
Iter-33, train loss: 30.68970072, valid loss: 1617.13208498
Iter-34, train loss: 28.13366481, valid loss: 1384.36507496
Iter-35, train loss: 29.61488682, valid loss: 1600.88942007
Iter-36, train loss: 29.52342062, valid loss: 1428.18786080
Iter-37, train loss: 30.71082591, valid loss: 1610.31203594
Iter-38, train loss: 30.13908023, valid loss: 1517.19237518
Iter-39, train loss: 30.46714373, valid loss: 1443.76473276
Iter-40, train loss: 30.78931562, valid loss: 1318.40553092
Iter-41, train loss: 30.74492751, valid loss: 1386.91678921
Iter-42, train loss: 30.58853798, valid loss: 1506.10640022
Iter-43, train loss: 29.32486102, valid loss: 1421.21443626
Iter-44, train loss: 30.26163250, valid loss: 1574.47792084
Iter-45, train loss: 29.80317360, valid loss: 1496.41661600
Iter-46, train loss: 29.08822349, valid loss: 1519.76080382
Iter-47, train loss: 28.62814727, valid loss: 1633.04218535
Iter-48, train loss: 29.58408503, valid loss: 1730.29120177
Iter-49, train loss: 29.16902783, valid loss: 1638.46444127
Iter-50, train loss: 28.87402210, valid loss: 1551.97269132
Iter-51, train loss: 30.50041045, valid loss: 1678.25273557
Iter-52, train loss: 29.86386084, valid loss: 1736.16861436
Iter-53, train loss: 30.79034656, valid loss: 1584.59940846
Iter-54, train loss: 30.11850848, valid loss: 1696.51445976
Iter-55, train loss: 25.74285954, valid loss: 1503.21849522
Iter-56, train loss: 30.20493981, valid loss: 1609.82926171
Iter-57, train loss: 29.48310244, valid loss: 1667.99345804
Iter-58, train loss: 29.85034760, valid loss: 1553.73492393
Iter-59, train loss: 30.31834623, valid loss: 1261.25428118
Iter-60, train loss: 28.90166624, valid loss: 1666.51091520
Iter-61, train loss: 29.64076880, valid loss: 1695.50818950
Iter-62, train loss: 30.69865939, valid loss: 1692.42506012
Iter-63, train loss: 26.77308644, valid loss: 1458.99692303
Iter-64, train loss: 30.06595871, valid loss: 1649.42629820
Iter-65, train loss: 30.14355733, valid loss: 1427.72059642
Iter-66, train loss: 26.32197551, valid loss: 1622.25068260
Iter-67, train loss: 30.03158844, valid loss: 1667.70659833
Iter-68, train loss: 28.77165561, valid loss: 1639.27862338
Iter-69, train loss: 29.56310546, valid loss: 1731.25385283
Iter-70, train loss: 30.53924956, valid loss: 1753.64376166
Iter-71, train loss: 30.93923264, valid loss: 1829.09967569
Iter-72, train loss: 29.71933559, valid loss: 1740.89577832
Iter-73, train loss: 30.45276101, valid loss: 1174.66748183
Iter-74, train loss: 30.30779711, valid loss: 1666.13086941
Iter-75, train loss: 29.62127209, valid loss: 1530.83442840
Iter-76, train loss: 29.25343599, valid loss: 1646.51370903
Iter-77, train loss: 30.34523715, valid loss: 1072.42215719
Iter-78, train loss: 30.58946584, valid loss: 1846.00019450
Iter-79, train loss: 29.86086151, valid loss: 1205.03140499
Iter-80, train loss: 30.15466612, valid loss: 1437.08582793
Iter-81, train loss: 29.82514081, valid loss: 1877.11250312
Iter-82, train loss: 28.73259527, valid loss: 871.71440604
Iter-83, train loss: 30.58294577, valid loss: 1443.89086673
Iter-84, train loss: 30.49360685, valid loss: 1662.54144454
Iter-85, train loss: 29.65134266, valid loss: 1120.04347202
Iter-86, train loss: 30.77443322, valid loss: 1670.49907732
Iter-87, train loss: 26.75285759, valid loss: 1563.88759106
Iter-88, train loss: 26.36317951, valid loss: 1777.07229663
Iter-89, train loss: 28.76625181, valid loss: 1168.05255839
Iter-90, train loss: 30.78931562, valid loss: 1113.93514433
Iter-91, train loss: 27.13605413, valid loss: 1341.87405657
Iter-92, train loss: 30.41128426, valid loss: 1586.10939897
Iter-93, train loss: 29.98536757, valid loss: 1757.39863101
Iter-94, train loss: 26.28571940, valid loss: 1409.49895962
Iter-95, train loss: 30.10814357, valid loss: 2029.95162420
Iter-96, train loss: 30.90828405, valid loss: 1556.29658211
Iter-97, train loss: 29.83859098, valid loss: 1284.01169055
Iter-98, train loss: 29.22941703, valid loss: 1959.75836959
Iter-99, train loss: 29.67157661, valid loss: 2013.95116278
Iter-100, train loss: 29.75157816, valid loss: 1381.16884034
Iter-101, train loss: 30.45040739, valid loss: 1049.14732885
Iter-102, train loss: 30.44287599, valid loss: 1648.92456006
Iter-103, train loss: 30.77631700, valid loss: 1694.50586557
Iter-104, train loss: 29.24716124, valid loss: 1619.14103687
Iter-105, train loss: 29.69452127, valid loss: 1800.23507390
Iter-106, train loss: 26.73575685, valid loss: 1944.77028853
Iter-107, train loss: 29.12600762, valid loss: 2299.04201316
Iter-108, train loss: 30.07537324, valid loss: 1953.82881990
Iter-109, train loss: 29.58222667, valid loss: 1727.58189486
Iter-110, train loss: 28.83019743, valid loss: 1940.20438938
Iter-111, train loss: 29.35126119, valid loss: 1741.32615145
Iter-112, train loss: 30.66018257, valid loss: 2045.55689924
Iter-113, train loss: 28.98386551, valid loss: 1476.55291232
Iter-114, train loss: 26.01849595, valid loss: 1277.31362964
Iter-115, train loss: 30.22143597, valid loss: 1298.37132579
Iter-116, train loss: 29.73276330, valid loss: 1024.53600263
Iter-117, train loss: 28.85220961, valid loss: 1797.47691720
Iter-118, train loss: 30.16580006, valid loss: 1247.32831646
Iter-119, train loss: 30.29369550, valid loss: 1705.96611543
Iter-120, train loss: 29.77578460, valid loss: 1471.00206355
Iter-121, train loss: 28.90885755, valid loss: 2328.03616552
Iter-122, train loss: 30.50637861, valid loss: 1826.40271228
Iter-123, train loss: 28.06648191, valid loss: 1893.55066079
Iter-124, train loss: 30.21816265, valid loss: 1556.12726575
Iter-125, train loss: 30.12421554, valid loss: 2143.45040585
Iter-126, train loss: 25.07286917, valid loss: 761.61737153
Iter-127, train loss: 30.11970573, valid loss: 1583.40657685
Iter-128, train loss: 29.23107741, valid loss: 1427.10019326
Iter-129, train loss: 30.54812736, valid loss: 1723.95902710
Iter-130, train loss: 29.95612798, valid loss: 1465.77180167
Iter-131, train loss: 29.88360036, valid loss: 1586.51345992
Iter-132, train loss: 29.81290529, valid loss: 1347.20255311
Iter-133, train loss: 29.03315264, valid loss: 1400.12226830
Iter-134, train loss: 29.90091667, valid loss: 719.92199970
Iter-135, train loss: 30.35269167, valid loss: 1817.04120355
Iter-136, train loss: 27.09814187, valid loss: 1249.57376075
Iter-137, train loss: 30.78931562, valid loss: 716.00762283
Iter-138, train loss: 27.03559108, valid loss: 1554.04867978
Iter-139, train loss: 30.24824187, valid loss: 1061.26775536
Iter-140, train loss: 29.62871563, valid loss: 1711.15606725
Iter-141, train loss: 29.80951457, valid loss: 764.73603055
Iter-142, train loss: 30.73653678, valid loss: 1536.13410695
Iter-143, train loss: 30.74123968, valid loss: 1241.55557409
Iter-144, train loss: 29.15266163, valid loss: 998.97290895
Iter-145, train loss: 30.50212991, valid loss: 1253.79626411
Iter-146, train loss: 30.55196102, valid loss: 980.18374431
Iter-147, train loss: 29.80841788, valid loss: 1232.87079362
Iter-148, train loss: 30.42500593, valid loss: 929.58891789
Iter-149, train loss: 29.60305027, valid loss: 1084.99549387
Iter-150, train loss: 29.75932734, valid loss: 1498.82339541
Iter-151, train loss: 28.51759269, valid loss: 1867.25590578
Iter-152, train loss: 30.78931562, valid loss: 1491.03763318
Iter-153, train loss: 25.32937650, valid loss: 877.32844311
Iter-154, train loss: 29.75007486, valid loss: 1186.78922005
Iter-155, train loss: 27.17068232, valid loss: 1241.60873563
Iter-156, train loss: 27.65340554, valid loss: 1571.57289722
Iter-157, train loss: 29.51604381, valid loss: 828.20651827
Iter-158, train loss: 28.74270441, valid loss: 929.96426236
Iter-159, train loss: 30.51244328, valid loss: 1324.27108920
Iter-160, train loss: 30.39812390, valid loss: 1961.78335639
Iter-161, train loss: 29.91192869, valid loss: 1077.71523928
Iter-162, train loss: 29.78727436, valid loss: 1042.47197535
Iter-163, train loss: 25.67363466, valid loss: 1114.13698956
Iter-164, train loss: 29.90524463, valid loss: 1129.27105503
Iter-165, train loss: 27.82136818, valid loss: 1540.14653890
Iter-166, train loss: 26.87614672, valid loss: 1098.48013756
Iter-167, train loss: 30.41596055, valid loss: 1349.21157330
Iter-168, train loss: 29.94049423, valid loss: 1629.76077134
Iter-169, train loss: 28.01218555, valid loss: 2058.05053243
Iter-170, train loss: 30.60924052, valid loss: 751.45370540
Iter-171, train loss: 29.24422887, valid loss: 1551.35771430
Iter-172, train loss: 26.75216393, valid loss: 2064.86828430
Iter-173, train loss: 30.22202804, valid loss: 1249.31400136
Iter-174, train loss: 30.53434348, valid loss: 1529.36252094
Iter-175, train loss: 28.98353980, valid loss: 1708.85049045
Iter-176, train loss: 30.83262354, valid loss: 1505.97805451
Iter-177, train loss: 29.71894248, valid loss: 1605.10007915
Iter-178, train loss: 29.78525191, valid loss: 1294.09957102
Iter-179, train loss: 29.67051525, valid loss: 872.58149715
Iter-180, train loss: 29.84364404, valid loss: 1190.00709160
Iter-181, train loss: 28.04380599, valid loss: 755.83765011
Iter-182, train loss: 30.30074035, valid loss: 1565.74671423
Iter-183, train loss: 30.47692747, valid loss: 979.49007267
Iter-184, train loss: 30.78931562, valid loss: 1826.91798428
Iter-185, train loss: 26.83048349, valid loss: 1804.97648814
Iter-186, train loss: 30.12449581, valid loss: 1408.09345905
Iter-187, train loss: 30.10472731, valid loss: 1524.02744743
Iter-188, train loss: 30.75515344, valid loss: 1249.43625834
Iter-189, train loss: 30.29325679, valid loss: 2461.32949124
Iter-190, train loss: 28.35508762, valid loss: 984.25686010
Iter-191, train loss: 30.74162773, valid loss: 1244.15159384
Iter-192, train loss: 26.77580354, valid loss: 1515.07184276
Iter-193, train loss: 26.75910919, valid loss: 829.73761220
Iter-194, train loss: 30.48190317, valid loss: 749.76257215
Iter-195, train loss: 26.32838596, valid loss: 1303.41457939
Iter-196, train loss: 29.49421306, valid loss: 1702.10760712
Iter-197, train loss: 29.74640949, valid loss: 1509.92871386
Iter-198, train loss: 29.72764337, valid loss: 816.45083110
Iter-199, train loss: 29.31745003, valid loss: 783.00433256
Iter-200, train loss: 30.79069276, valid loss: 777.43010790
Iter-201, train loss: 30.13061086, valid loss: 1171.72476993
Iter-202, train loss: 29.68053186, valid loss: 1513.53739056
Iter-203, train loss: 30.32703780, valid loss: 852.77726519
Iter-204, train loss: 29.58545163, valid loss: 1787.94802290
Iter-205, train loss: 29.72391467, valid loss: 855.83011851
Iter-206, train loss: 30.63099343, valid loss: 1091.68927668
Iter-207, train loss: 29.94472433, valid loss: 1006.17528412
Iter-208, train loss: 30.78931562, valid loss: 1151.16703953
Iter-209, train loss: 26.22320793, valid loss: 1892.03783915
Iter-210, train loss: 30.78931562, valid loss: 1244.98234942
Iter-211, train loss: 30.12666929, valid loss: 937.52712882
Iter-212, train loss: 29.77035147, valid loss: 1267.53924859
Iter-213, train loss: 29.53844327, valid loss: 932.02691184
Iter-214, train loss: 30.27564897, valid loss: 1065.40938489
Iter-215, train loss: 30.78931562, valid loss: 1775.88884839
Iter-216, train loss: 25.90437799, valid loss: 2124.54846569
Iter-217, train loss: 25.97576874, valid loss: 1208.26260879
Iter-218, train loss: 29.55824136, valid loss: 914.15151415
Iter-219, train loss: 30.78931562, valid loss: 814.64826088
Iter-220, train loss: 29.37594936, valid loss: 1219.50083936
Iter-221, train loss: 30.96665257, valid loss: 1263.95057582
Iter-222, train loss: 28.07628427, valid loss: 1238.82185952
Iter-223, train loss: 26.79142063, valid loss: 896.81579481
Iter-224, train loss: 30.14011632, valid loss: 1640.73780754
Iter-225, train loss: 30.57487160, valid loss: 1650.21627512
Iter-226, train loss: 30.37458397, valid loss: 849.76151115
Iter-227, train loss: 30.24641836, valid loss: 973.41015568
Iter-228, train loss: 30.13190568, valid loss: 1414.44214004
Iter-229, train loss: 29.46710740, valid loss: 1020.39219492
Iter-230, train loss: 26.43253372, valid loss: 1442.26645121
Iter-231, train loss: 30.63778476, valid loss: 967.17592115
Iter-232, train loss: 27.13672149, valid loss: 1432.96599359
Iter-233, train loss: 30.33764101, valid loss: 1224.85223600
Iter-234, train loss: 30.56837903, valid loss: 965.93213554
Iter-235, train loss: 28.26997245, valid loss: 1305.84762606
Iter-236, train loss: 29.36404905, valid loss: 1254.92812650
Iter-237, train loss: 22.56881331, valid loss: 835.32634347
Iter-238, train loss: 30.67576064, valid loss: 1232.95557401
Iter-239, train loss: 30.10642274, valid loss: 875.89119614
Iter-240, train loss: 30.78931562, valid loss: 949.10484441
Iter-241, train loss: 20.68254044, valid loss: 1436.42212136
Iter-242, train loss: 30.27856082, valid loss: 1239.52183972
Iter-243, train loss: 26.80526691, valid loss: 1073.90917070
Iter-244, train loss: 29.84585765, valid loss: 1142.33080568
Iter-245, train loss: 29.42960656, valid loss: 1483.48601525
Iter-246, train loss: 30.77490725, valid loss: 1196.24946229
Iter-247, train loss: 29.87647587, valid loss: 931.33582058
Iter-248, train loss: 26.44514466, valid loss: 782.63882579
Iter-249, train loss: 29.80801263, valid loss: 1207.12470451
Iter-250, train loss: 29.84531109, valid loss: 1174.66069760
Iter-251, train loss: 30.32023000, valid loss: 1401.66437049
Iter-252, train loss: 30.64321522, valid loss: 1806.86484911
Iter-253, train loss: 28.85789795, valid loss: 1040.66052416
Iter-254, train loss: 27.58464250, valid loss: 1893.15557345
Iter-255, train loss: 30.32559399, valid loss: 1242.37292575
Iter-256, train loss: 30.64028947, valid loss: 1227.84765440
Iter-257, train loss: 30.08199465, valid loss: 1020.43163854
Iter-258, train loss: 29.25330327, valid loss: 1098.58454442
Iter-259, train loss: 29.37303812, valid loss: 2056.65549533
Iter-260, train loss: 23.99851358, valid loss: 1757.43051112
Iter-261, train loss: 27.78939147, valid loss: 1291.21665928
Iter-262, train loss: 29.53005909, valid loss: 812.43449566
Iter-263, train loss: 30.59578076, valid loss: 1262.57506336
Iter-264, train loss: 28.80289596, valid loss: 1093.09972804
Iter-265, train loss: 28.67910291, valid loss: 1098.32606198
Iter-266, train loss: 29.78771750, valid loss: 1499.41580956
Iter-267, train loss: 29.38266583, valid loss: 1410.21796297
Iter-268, train loss: 29.26143100, valid loss: 1398.66891677
Iter-269, train loss: 28.55661082, valid loss: 1105.54181326
Iter-270, train loss: 30.76139428, valid loss: 1065.45412407
Iter-271, train loss: 30.09100119, valid loss: 1026.87782950
Iter-272, train loss: 30.77344029, valid loss: 1516.24772941
Iter-273, train loss: 30.78931562, valid loss: 1198.19081770
Iter-274, train loss: 28.89716272, valid loss: 1170.16697330
Iter-275, train loss: 29.71533705, valid loss: 1119.24204788
Iter-276, train loss: 30.18571233, valid loss: 590.60959926
Iter-277, train loss: 29.70481643, valid loss: 981.27021426
Iter-278, train loss: 29.24742345, valid loss: 919.20168351
Iter-279, train loss: 30.63861027, valid loss: 1072.92191846
Iter-280, train loss: 30.35751985, valid loss: 1269.96744112
Iter-281, train loss: 28.93764069, valid loss: 1162.79667159
Iter-282, train loss: 29.58215405, valid loss: 967.44116824
Iter-283, train loss: 26.94059208, valid loss: 1742.35231008
Iter-284, train loss: 26.10891935, valid loss: 884.23058044
Iter-285, train loss: 28.99210446, valid loss: 1130.61175189
Iter-286, train loss: 30.46279173, valid loss: 1369.82640574
Iter-287, train loss: 30.63145485, valid loss: 1246.73509067
Iter-288, train loss: 29.02522279, valid loss: 1008.32078841
Iter-289, train loss: 29.90671497, valid loss: 1345.48397575
Iter-290, train loss: 29.99760870, valid loss: 921.78333342
Iter-291, train loss: 29.89194086, valid loss: 1233.72356276
Iter-292, train loss: 27.58246884, valid loss: 1051.82490226
Iter-293, train loss: 29.74479062, valid loss: 1286.87514448
Iter-294, train loss: 27.94738366, valid loss: 1798.71053930
Iter-295, train loss: 30.40789876, valid loss: 1031.41546831
Iter-296, train loss: 30.78931562, valid loss: 1117.01019716
Iter-297, train loss: 30.71414975, valid loss: 851.16037940
Iter-298, train loss: 30.44122277, valid loss: 1087.52499868
Iter-299, train loss: 29.27451924, valid loss: 941.72857039
Iter-300, train loss: 29.95601664, valid loss: 1005.45147240
Iter-301, train loss: 29.28047538, valid loss: 1263.17160401
Iter-302, train loss: 27.51118986, valid loss: 1107.14486904
Iter-303, train loss: 26.20486943, valid loss: 825.13762631
Iter-304, train loss: 29.30975927, valid loss: 1575.92162649
Iter-305, train loss: 26.24028838, valid loss: 1068.28268225
Iter-306, train loss: 26.44439830, valid loss: 872.88816186
Iter-307, train loss: 30.13830056, valid loss: 1070.60950456
Iter-308, train loss: 29.72805063, valid loss: 792.07677557
Iter-309, train loss: 26.66665188, valid loss: 1251.35618048
Iter-310, train loss: 29.56948216, valid loss: 1128.13467080
Iter-311, train loss: 30.48401177, valid loss: 954.44742900
Iter-312, train loss: 27.11639201, valid loss: 778.04951483
Iter-313, train loss: 30.73390510, valid loss: 1000.75227394
Iter-314, train loss: 27.41019871, valid loss: 1050.37636541
Iter-315, train loss: 28.17656182, valid loss: 1137.93469666
Iter-316, train loss: 29.44127054, valid loss: 1053.92324782
Iter-317, train loss: 30.47275051, valid loss: 1037.57959329
Iter-318, train loss: 30.30870987, valid loss: 786.04640400
Iter-319, train loss: 30.61774575, valid loss: 958.82293196
Iter-320, train loss: 30.09731029, valid loss: 971.66872828
Iter-321, train loss: 29.71029478, valid loss: 756.80182435
Iter-322, train loss: 30.33543538, valid loss: 1028.95571809
Iter-323, train loss: 30.28922844, valid loss: 1545.15890444
Iter-324, train loss: 29.26769335, valid loss: 1298.03580374
Iter-325, train loss: 30.31220505, valid loss: 1088.38910944
Iter-326, train loss: 30.10329984, valid loss: 940.79806402
Iter-327, train loss: 30.78931562, valid loss: 765.46659118
Iter-328, train loss: 29.20932974, valid loss: 1103.51746190
Iter-329, train loss: 29.18143199, valid loss: 918.24667004
Iter-330, train loss: 30.49395951, valid loss: 1365.95415521
Iter-331, train loss: 30.78931562, valid loss: 988.72732414
Iter-332, train loss: 30.07793232, valid loss: 1161.97542020
Iter-333, train loss: 29.76842067, valid loss: 1100.74650660
Iter-334, train loss: 29.77350195, valid loss: 1292.61177750
Iter-335, train loss: 27.86430296, valid loss: 945.76307298
Iter-336, train loss: 27.99627639, valid loss: 1436.67618269
Iter-337, train loss: 30.70221784, valid loss: 1072.86835895
Iter-338, train loss: 29.18203036, valid loss: 1120.31305547
Iter-339, train loss: 30.78931562, valid loss: 1196.55694168
Iter-340, train loss: 30.79520317, valid loss: 732.71112865
Iter-341, train loss: 24.63429606, valid loss: 1406.79615525
Iter-342, train loss: 29.26413119, valid loss: 1028.35323533
Iter-343, train loss: 30.14121376, valid loss: 1492.71980443
Iter-344, train loss: 29.02781110, valid loss: 890.29905064
Iter-345, train loss: 26.60237291, valid loss: 1083.97762526
Iter-346, train loss: 29.78367306, valid loss: 869.50396606
Iter-347, train loss: 29.67722016, valid loss: 990.97747014
Iter-348, train loss: 29.86449991, valid loss: 1269.38000482
Iter-349, train loss: 29.29102444, valid loss: 859.83761329
Iter-350, train loss: 28.68970567, valid loss: 987.03817433
Iter-351, train loss: 28.83519103, valid loss: 1133.23028490
Iter-352, train loss: 30.53216675, valid loss: 677.01262580
Iter-353, train loss: 26.97940224, valid loss: 1566.04312126
Iter-354, train loss: 29.24556628, valid loss: 874.49642849
Iter-355, train loss: 26.36536239, valid loss: 1683.77404686
Iter-356, train loss: 30.78931562, valid loss: 1079.81782377
Iter-357, train loss: 30.18967999, valid loss: 1067.64752030
Iter-358, train loss: 29.28680186, valid loss: 928.67446616
Iter-359, train loss: 29.54841422, valid loss: 1231.07993459
Iter-360, train loss: 29.49681599, valid loss: 863.59570381
Iter-361, train loss: 28.59088837, valid loss: 791.37428394
Iter-362, train loss: 30.78672473, valid loss: 1120.10155520
Iter-363, train loss: 30.78931562, valid loss: 993.81156411
Iter-364, train loss: 30.56331855, valid loss: 935.83851464
Iter-365, train loss: 29.63013500, valid loss: 775.83938480
Iter-366, train loss: 28.12462480, valid loss: 1140.84646642
Iter-367, train loss: 26.79043267, valid loss: 832.12367952
Iter-368, train loss: 30.62041051, valid loss: 842.09869187
Iter-369, train loss: 30.13726200, valid loss: 1001.01184174
Iter-370, train loss: 30.08710997, valid loss: 1288.14051349
Iter-371, train loss: 27.45076359, valid loss: 961.13077461
Iter-372, train loss: 30.13173217, valid loss: 795.71172832
Iter-373, train loss: 30.27268621, valid loss: 929.45767783
Iter-374, train loss: 24.55463104, valid loss: 1158.43936039
Iter-375, train loss: 29.77354465, valid loss: 1135.44377620
Iter-376, train loss: 30.62022261, valid loss: 834.90300016
Iter-377, train loss: 30.78931562, valid loss: 847.08646163
Iter-378, train loss: 30.31165711, valid loss: 1108.36320747
Iter-379, train loss: 26.16544834, valid loss: 1250.63025378
Iter-380, train loss: 29.73616157, valid loss: 1163.66843254
Iter-381, train loss: 29.71149665, valid loss: 929.49427022
Iter-382, train loss: 28.49655620, valid loss: 732.87388922
Iter-383, train loss: 29.56753753, valid loss: 1065.61010349
Iter-384, train loss: 30.53155598, valid loss: 1168.44575434
Iter-385, train loss: 29.97885376, valid loss: 785.86671415
Iter-386, train loss: 30.44947689, valid loss: 972.77975778
Iter-387, train loss: 30.76269020, valid loss: 1030.18807335
Iter-388, train loss: 29.87723868, valid loss: 855.60564202
Iter-389, train loss: 30.03735823, valid loss: 741.28584151
Iter-390, train loss: 29.27956865, valid loss: 1288.89608598
Iter-391, train loss: 30.78931562, valid loss: 816.91677039
Iter-392, train loss: 30.73558393, valid loss: 910.28815240
Iter-393, train loss: 29.22721294, valid loss: 908.96525221
Iter-394, train loss: 30.14506284, valid loss: 899.10487056
Iter-395, train loss: 30.08538061, valid loss: 1280.52298606
Iter-396, train loss: 29.50424344, valid loss: 1141.00823085
Iter-397, train loss: 29.29464495, valid loss: 1058.02956365
Iter-398, train loss: 30.38391881, valid loss: 930.53606956
Iter-399, train loss: 26.67195764, valid loss: 904.23710732
Iter-400, train loss: 29.86696982, valid loss: 790.56404137
Iter-401, train loss: 30.69008009, valid loss: 944.73833280
Iter-402, train loss: 29.83523518, valid loss: 837.10048892
Iter-403, train loss: 29.84575375, valid loss: 1081.81183588
Iter-404, train loss: 30.78594016, valid loss: 1051.18921885
Iter-405, train loss: 30.58963878, valid loss: 1366.79647616
Iter-406, train loss: 30.78931562, valid loss: 1323.02649206
Iter-407, train loss: 29.65385540, valid loss: 1164.99377896
Iter-408, train loss: 30.06536746, valid loss: 1066.81449444
Iter-409, train loss: 29.71001541, valid loss: 796.24744397
Iter-410, train loss: 29.74913499, valid loss: 1057.31851847
Iter-411, train loss: 29.19972947, valid loss: 1384.28436842
Iter-412, train loss: 29.38663654, valid loss: 898.01164273
Iter-413, train loss: 29.73387197, valid loss: 1122.98714240
Iter-414, train loss: 29.19772763, valid loss: 968.89295514
Iter-415, train loss: 29.56803530, valid loss: 1811.47582916
Iter-416, train loss: 29.78777235, valid loss: 1301.95520317
Iter-417, train loss: 27.92753422, valid loss: 991.31155538
Iter-418, train loss: 29.23099942, valid loss: 874.02734993
Iter-419, train loss: 27.36730301, valid loss: 1206.71186196
Iter-420, train loss: 29.31704812, valid loss: 1098.88306795
Iter-421, train loss: 26.18849512, valid loss: 1496.80178272
Iter-422, train loss: 30.71810020, valid loss: 902.34922759
Iter-423, train loss: 29.41112929, valid loss: 1097.18387215
Iter-424, train loss: 30.88319097, valid loss: 826.11105087
Iter-425, train loss: 27.24840009, valid loss: 1085.36898059
Iter-426, train loss: 29.20118782, valid loss: 1257.65474023
Iter-427, train loss: 30.78931562, valid loss: 1020.18807057
Iter-428, train loss: 27.18191236, valid loss: 1001.64649108
Iter-429, train loss: 24.66650414, valid loss: 990.99487089
Iter-430, train loss: 30.15919108, valid loss: 982.63369254
Iter-431, train loss: 30.04665668, valid loss: 1104.52172939
Iter-432, train loss: 29.23053878, valid loss: 1038.79958235
Iter-433, train loss: 30.54827401, valid loss: 814.99819623
Iter-434, train loss: 30.23093062, valid loss: 920.41397868
Iter-435, train loss: 25.59561445, valid loss: 885.96316741
Iter-436, train loss: 29.14366109, valid loss: 1186.18257927
Iter-437, train loss: 30.84362942, valid loss: 1602.86908360
Iter-438, train loss: 30.21045211, valid loss: 924.89217828
Iter-439, train loss: 29.74686453, valid loss: 948.34440384
Iter-440, train loss: 30.36424827, valid loss: 991.02789256
Iter-441, train loss: 30.48219005, valid loss: 1085.32809947
Iter-442, train loss: 30.49867916, valid loss: 1128.44330094
Iter-443, train loss: 30.72701390, valid loss: 933.68948885
Iter-444, train loss: 25.04972784, valid loss: 944.19470877
Iter-445, train loss: 29.20068408, valid loss: 1241.11638768
Iter-446, train loss: 30.64740484, valid loss: 882.62075590
Iter-447, train loss: 30.78931562, valid loss: 1161.98644046
Iter-448, train loss: 28.92889962, valid loss: 1144.43071963
Iter-449, train loss: 29.07640477, valid loss: 1271.37684782
Iter-450, train loss: 29.63834626, valid loss: 990.08049489
Iter-451, train loss: 30.75891132, valid loss: 957.70969750
Iter-452, train loss: 30.23376370, valid loss: 1273.59127356
Iter-453, train loss: 29.33363252, valid loss: 1064.30767203
Iter-454, train loss: 25.44980291, valid loss: 1068.86042479
Iter-455, train loss: 30.13528841, valid loss: 1308.50646503
Iter-456, train loss: 29.84873696, valid loss: 958.33006167
Iter-457, train loss: 30.80187650, valid loss: 1128.98230527
Iter-458, train loss: 30.21163712, valid loss: 1518.88922279
Iter-459, train loss: 26.82609402, valid loss: 1225.36590218
Iter-460, train loss: 29.15320531, valid loss: 919.64390210
Iter-461, train loss: 30.49799350, valid loss: 1498.56766629
Iter-462, train loss: 29.50083825, valid loss: 1008.36916929
Iter-463, train loss: 24.87553988, valid loss: 893.89168819
Iter-464, train loss: 30.78292775, valid loss: 1310.10836264
Iter-465, train loss: 29.03378686, valid loss: 1019.64008508
Iter-466, train loss: 26.26765288, valid loss: 1149.93406283
Iter-467, train loss: 27.39927029, valid loss: 965.14729426
Iter-468, train loss: 30.18099346, valid loss: 1144.23103349
Iter-469, train loss: 29.31800439, valid loss: 764.73371361
Iter-470, train loss: 28.83790866, valid loss: 1036.18022656
Iter-471, train loss: 29.67537644, valid loss: 880.25711614
Iter-472, train loss: 28.63017540, valid loss: 1104.71293110
Iter-473, train loss: 29.99469151, valid loss: 1445.09812857
Iter-474, train loss: 29.96472598, valid loss: 1049.53307141
Iter-475, train loss: 29.83117135, valid loss: 1491.38336420
Iter-476, train loss: 29.34788437, valid loss: 1397.68360552
Iter-477, train loss: 30.01062143, valid loss: 1427.42760028
Iter-478, train loss: 30.08445966, valid loss: 884.22870735
Iter-479, train loss: 28.85886664, valid loss: 1088.64061614
Iter-480, train loss: 30.35526560, valid loss: 939.45548643
Iter-481, train loss: 29.28496278, valid loss: 970.87720449
Iter-482, train loss: 30.09194474, valid loss: 757.10340110
Iter-483, train loss: 29.98931670, valid loss: 1657.62827627
Iter-484, train loss: 30.09528408, valid loss: 1307.87440978
Iter-485, train loss: 30.28030002, valid loss: 894.62015810
Iter-486, train loss: 30.03281742, valid loss: 1350.87581372
Iter-487, train loss: 26.15025973, valid loss: 904.72250472
Iter-488, train loss: 28.41604430, valid loss: 1268.32304729
Iter-489, train loss: 29.04321636, valid loss: 1848.85593867
Iter-490, train loss: 29.96289674, valid loss: 1428.00957109
Iter-491, train loss: 29.59859745, valid loss: 1116.78272585
Iter-492, train loss: 29.65391250, valid loss: 1166.50727464
Iter-493, train loss: 29.40800726, valid loss: 1020.34609195
Iter-494, train loss: 28.22143854, valid loss: 1225.25469432
Iter-495, train loss: 30.23820373, valid loss: 1129.05608629
Iter-496, train loss: 28.34843583, valid loss: 740.02917986
Iter-497, train loss: 29.78245469, valid loss: 1046.64484719
Iter-498, train loss: 29.84376064, valid loss: 1012.16218815
Iter-499, train loss: 29.70151060, valid loss: 1225.34374189
Iter-500, train loss: 28.70850497, valid loss: 860.35739789
Iter-501, train loss: 27.99008135, valid loss: 1205.48846570
Iter-502, train loss: 24.01615557, valid loss: 951.22027061
Iter-503, train loss: 30.52308528, valid loss: 847.55542129
Iter-504, train loss: 29.55615410, valid loss: 868.81380371
Iter-505, train loss: 28.74640578, valid loss: 937.15278283
Iter-506, train loss: 30.22095838, valid loss: 1257.74713326
Iter-507, train loss: 27.90495946, valid loss: 827.07498370
Iter-508, train loss: 26.90350164, valid loss: 990.60325485
Iter-509, train loss: 27.87203149, valid loss: 1158.06189459
Iter-510, train loss: 30.22828299, valid loss: 1517.48238472
Iter-511, train loss: 30.64227321, valid loss: 1127.74499646
Iter-512, train loss: 30.15121663, valid loss: 1033.45906852
Iter-513, train loss: 29.71139408, valid loss: 2020.30942184
Iter-514, train loss: 27.49738844, valid loss: 1232.80015737
Iter-515, train loss: 30.29177324, valid loss: 1087.71067489
Iter-516, train loss: 30.12364337, valid loss: 1043.79840459
Iter-517, train loss: 29.83518952, valid loss: 1172.24373757
Iter-518, train loss: 30.19314753, valid loss: 897.82043029
Iter-519, train loss: 30.16974907, valid loss: 979.81596830
Iter-520, train loss: 30.78931562, valid loss: 1413.97222622
Iter-521, train loss: 30.56179437, valid loss: 1395.56844575
Iter-522, train loss: 30.39182731, valid loss: 1063.98724410
Iter-523, train loss: 27.36121750, valid loss: 1001.81251324
Iter-524, train loss: 30.23527133, valid loss: 844.81268705
Iter-525, train loss: 30.34811292, valid loss: 1955.85222924
Iter-526, train loss: 30.37344720, valid loss: 978.81950325
Iter-527, train loss: 29.34947258, valid loss: 1311.87547524
Iter-528, train loss: 27.89822629, valid loss: 1160.28553306
Iter-529, train loss: 29.77730299, valid loss: 1001.38666496
Iter-530, train loss: 30.23002780, valid loss: 1322.62669974
Iter-531, train loss: 30.76735559, valid loss: 1165.05603525
Iter-532, train loss: 30.44940303, valid loss: 1223.75166188
Iter-533, train loss: 30.20657663, valid loss: 1109.58430786
Iter-534, train loss: 28.66696057, valid loss: 1012.26984413
Iter-535, train loss: 27.28548706, valid loss: 1422.07031365
Iter-536, train loss: 29.89013848, valid loss: 1039.04009495
Iter-537, train loss: 29.77871186, valid loss: 1265.05520556
Iter-538, train loss: 29.11920327, valid loss: 1752.36215204
Iter-539, train loss: 26.62746936, valid loss: 1003.87520024
Iter-540, train loss: 29.79746832, valid loss: 944.82661597
Iter-541, train loss: 30.62083050, valid loss: 1128.81756509
Iter-542, train loss: 29.27341136, valid loss: 1290.75790794
Iter-543, train loss: 29.77031907, valid loss: 1375.50628487
Iter-544, train loss: 29.23016393, valid loss: 1055.98823528
Iter-545, train loss: 29.59112300, valid loss: 1162.39119478
Iter-546, train loss: 29.34041669, valid loss: 1032.34480841
Iter-547, train loss: 30.32860075, valid loss: 878.08678678
Iter-548, train loss: 28.99259972, valid loss: 1318.96907784
Iter-549, train loss: 28.42103684, valid loss: 1363.71141985
Iter-550, train loss: 29.42080040, valid loss: 1086.01518867
Iter-551, train loss: 26.15542915, valid loss: 1116.82371028
Iter-552, train loss: 26.72672130, valid loss: 990.41277247
Iter-553, train loss: 27.09671095, valid loss: 1009.91124915
Iter-554, train loss: 29.11308457, valid loss: 981.40726303
Iter-555, train loss: 30.35443267, valid loss: 1328.78415129
Iter-556, train loss: 28.01090841, valid loss: 1246.82730093
Iter-557, train loss: 28.20815457, valid loss: 989.51798976
Iter-558, train loss: 30.36381799, valid loss: 1236.31813702
Iter-559, train loss: 30.15439302, valid loss: 931.98458774
Iter-560, train loss: 30.33609572, valid loss: 1092.71340468
Iter-561, train loss: 29.42467521, valid loss: 858.53931620
Iter-562, train loss: 30.30242552, valid loss: 820.10861411
Iter-563, train loss: 29.33377098, valid loss: 981.76611005
Iter-564, train loss: 30.49765062, valid loss: 1001.02834035
Iter-565, train loss: 29.46794735, valid loss: 1547.14192685
Iter-566, train loss: 26.18947338, valid loss: 897.18815505
Iter-567, train loss: 28.53050051, valid loss: 1206.06482698
Iter-568, train loss: 30.78931562, valid loss: 1324.32022641
Iter-569, train loss: 26.69967550, valid loss: 1302.48645784
Iter-570, train loss: 29.72856101, valid loss: 1338.91073298
Iter-571, train loss: 26.80371917, valid loss: 1288.77237630
Iter-572, train loss: 30.11417382, valid loss: 1021.01372120
Iter-573, train loss: 27.22333183, valid loss: 1233.31370918
Iter-574, train loss: 29.03366241, valid loss: 869.79487794
Iter-575, train loss: 30.14679360, valid loss: 1132.78043282
Iter-576, train loss: 29.73206697, valid loss: 1098.90935854
Iter-577, train loss: 30.48336174, valid loss: 1040.12481983
Iter-578, train loss: 30.78931562, valid loss: 1326.83544716
Iter-579, train loss: 30.38067637, valid loss: 1134.14851088
Iter-580, train loss: 30.32883728, valid loss: 1271.42622128
Iter-581, train loss: 29.54646647, valid loss: 1100.61866160
Iter-582, train loss: 30.18264431, valid loss: 1171.94368462
Iter-583, train loss: 29.45716266, valid loss: 857.43881812
Iter-584, train loss: 28.92471746, valid loss: 1321.17740374
Iter-585, train loss: 27.52764186, valid loss: 1055.18895082
Iter-586, train loss: 28.95319528, valid loss: 1207.94605122
Iter-587, train loss: 30.36648205, valid loss: 1151.43602542
Iter-588, train loss: 30.13559717, valid loss: 1418.40557555
Iter-589, train loss: 25.85201649, valid loss: 1391.76543885
Iter-590, train loss: 30.05597269, valid loss: 1299.51777661
Iter-591, train loss: 30.72489936, valid loss: 1087.63996273
Iter-592, train loss: 30.00059070, valid loss: 1127.62251293
Iter-593, train loss: 29.01362465, valid loss: 1336.90997595
Iter-594, train loss: 29.73204039, valid loss: 1132.36296039
Iter-595, train loss: 30.54233379, valid loss: 1436.53174235
Iter-596, train loss: 27.25461224, valid loss: 1676.79391938
Iter-597, train loss: 30.34018751, valid loss: 1210.54382802
Iter-598, train loss: 28.37399134, valid loss: 1189.37439814
Iter-599, train loss: 29.95279932, valid loss: 1333.99875849
Iter-600, train loss: 29.06245997, valid loss: 955.57484138
Iter-601, train loss: 26.77448974, valid loss: 1745.47113428
Iter-602, train loss: 25.17367923, valid loss: 1079.21824828

In [ ]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['train'], label='Train loss')
plt.plot(net.losses['smooth train'], label='Train smooth loss')
plt.legend()
plt.show()


    

In [ ]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['valid'], label='Validation loss')
plt.legend()
plt.show()


    

In [ ]:

    

import matplotlib.pyplot as plt

y_pred = net.test(X_seed=X_valid[0], h=net.initial_state(), size=X_valid.shape[0]) # ys_tx1xn
y_pred.shape, Y_valid.shape

plt.plot(y_pred[:100, 0], label='y_pred')
plt.plot(Y_valid[:100, 0], label='Y_valid')
# plt.plot(X_valid[:100], label='X_valid')
plt.legend()
plt.show()


    

In [ ]:

    




    

In [ ]: