In [1]:

    

# Data: time-serie data from smartwatch or smartwatch data
# %matplotlib inline # for plt.show()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Data reading
# The smartwatch historical/time-seris data to visualize
# data_path = 'data/smartwatch_data/experimental_data_analysis/Basis_Watch_Data.csv'
# data_path = 'data/financial_data/USD_INR.csv'
data_path = 'data/bike_data/hour.csv'
data = pd.read_csv(data_path)

# Data: cleaning
# Getting rid of NaN
data = data.fillna(value=0.0)

# Showing the data file csv or comma separated value
data[:10]


    

    
Out[1]:







  

    

      

      
instant
      
dteday
      
season
      
yr
      
mnth
      
hr
      
holiday
      
weekday
      
workingday
      
weathersit
      
temp
      
atemp
      
hum
      
windspeed
      
casual
      
registered
      
cnt
    
  
  

    

      
0
      
1
      
2011-01-01
      
1
      
0
      
1
      
0
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.81
      
0.0000
      
3
      
13
      
16
    
    

      
1
      
2
      
2011-01-01
      
1
      
0
      
1
      
1
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
8
      
32
      
40
    
    

      
2
      
3
      
2011-01-01
      
1
      
0
      
1
      
2
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
5
      
27
      
32
    
    

      
3
      
4
      
2011-01-01
      
1
      
0
      
1
      
3
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
3
      
10
      
13
    
    

      
4
      
5
      
2011-01-01
      
1
      
0
      
1
      
4
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
0
      
1
      
1
    
    

      
5
      
6
      
2011-01-01
      
1
      
0
      
1
      
5
      
0
      
6
      
0
      
2
      
0.24
      
0.2576
      
0.75
      
0.0896
      
0
      
1
      
1
    
    

      
6
      
7
      
2011-01-01
      
1
      
0
      
1
      
6
      
0
      
6
      
0
      
1
      
0.22
      
0.2727
      
0.80
      
0.0000
      
2
      
0
      
2
    
    

      
7
      
8
      
2011-01-01
      
1
      
0
      
1
      
7
      
0
      
6
      
0
      
1
      
0.20
      
0.2576
      
0.86
      
0.0000
      
1
      
2
      
3
    
    

      
8
      
9
      
2011-01-01
      
1
      
0
      
1
      
8
      
0
      
6
      
0
      
1
      
0.24
      
0.2879
      
0.75
      
0.0000
      
1
      
7
      
8
    
    

      
9
      
10
      
2011-01-01
      
1
      
0
      
1
      
9
      
0
      
6
      
0
      
1
      
0.32
      
0.3485
      
0.76
      
0.0000
      
8
      
6
      
14
    
  

In [2]:

    

# # Plotting the smartwatch data before scaling/batch normalization
# data[:10000]['Price'].plot()
data[: 10].plot()
plt.legend()
plt.show()


    

In [3]:

    

data_array = np.array(data)
data_array.shape, data_array.dtype
data_main = np.array(data_array[:, -1:], dtype=float)
data_main.shape, data_main.dtype

plt.plot(data_main[:100])
plt.show()


    

In [4]:

    

mean = np.mean(data_main, axis=0)
std = np.std(data_main, axis=0)
std.shape, mean.shape, std.dtype, mean.dtype

data_norm = (data_main - mean) / std
plt.plot(data_norm[:100])
plt.show()
data_norm.mean(), data_norm.std(), data_norm.var(), data_norm.shape, data_norm.dtype


    

    













    
Out[4]:





(-1.0548364452851478e-16, 1.0, 1.0, (17379, 1), dtype('float64'))

In [5]:

    

train_data = data_norm[:16000] # the last dim/variable/feature
test_data = data_norm[16000:] # the last dim/variable/feature
train_data.shape, test_data.shape
X_train = train_data[0:15999]
Y_train = train_data[1:16000]
X_train.shape, Y_train.shape

plt.plot(X_train[:100])
plt.plot(Y_train[:100])
plt.show()


    

In [6]:

    

X_valid = test_data[0:1378] 
Y_valid = test_data[1:1379]
X_valid.shape, Y_valid.shape
plt.plot(X_valid[:100])
plt.plot(Y_valid[:100])
plt.show()


    

In [7]:

    

# Model or Network
import impl.layer as l
from impl.loss import *

class GRU:
    def __init__(self, D, H, p_dropout):
        self.D = D
        self.H = H
        self.p_dropout = p_dropout
        self.losses = {'train':[], 'smooth train':[], 'valid': []}
        
        # Model params
        Z = H + D
        m = dict(
            Wz=np.random.randn(Z, H) / np.sqrt(Z / 2.),
            Wr=np.random.randn(Z, H) / np.sqrt(Z / 2.),
            Wh=np.random.randn(Z, H) / np.sqrt(Z / 2.),
            Wy=np.random.randn(H, D) / np.sqrt(H / 2.),
            bz=np.zeros((1, H)),
            br=np.zeros((1, H)),
            bh=np.zeros((1, H)),
            by=np.zeros((1, D))
        )
        self.model = m
        
    def initial_state(self):
        return np.zeros((1, self.H))
    
    # The original implementatopn of forward pass 
    def forward(self, X_one_hot, h_old, m):
        Wz, Wr, Wh, Wy = m['Wz'], m['Wr'], m['Wh'], m['Wy']
        bz, br, bh, by = m['bz'], m['br'], m['bh'], m['by']

        X = np.column_stack((h_old, X_one_hot))

        hz, hz_cache = l.fc_forward(X, Wz, bz)
        hz, hz_sigm_cache = l.sigmoid_forward(hz)

        hr, hr_cache = l.fc_forward(X, Wr, br)
        hr, hr_sigm_cache = l.sigmoid_forward(hr)

        X_prime = np.column_stack((hr * h_old, X_one_hot))
        hh, hh_cache = l.fc_forward(X_prime, Wh, bh)
        hh, hh_tanh_cache = l.tanh_forward(hh)

        h = (1. - hz) * h_old + hz * hh

        y, y_cache = l.fc_forward(h, Wy, by)

        cache = (
            X, X_prime, h_old, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache,
            hh, hh_cache, hh_tanh_cache, h, y_cache
        )

        return y, h, cache

    # The original implementatopn of backward pass 
    def backward(self, dy, dh_next, cache):
        X, X_prime, h_old, hz, hz_cache, hz_sigm_cache, hr, hr_cache, hr_sigm_cache, hh, hh_cache, hh_tanh_cache, h, y_cache = cache

        dh, dWy, dby = l.fc_backward(dy, y_cache)
        dh += dh_next

        dhh = hz * dh
        dh_old1 = (1. - hz) * dh
        dhz = hh * dh - h_old * dh

        dhh = l.tanh_backward(dhh, hh_tanh_cache)
        dX_prime, dWh, dbh = l.fc_backward(dhh, hh_cache)

        dh_prime = dX_prime[:, :self.H]
        dh_old2 = hr * dh_prime

        dhr = h_old * dh_prime
        dhr = l.sigmoid_backward(dhr, hr_sigm_cache)
        dXr, dWr, dbr = l.fc_backward(dhr, hr_cache)

        dhz = l.sigmoid_backward(dhz, hz_sigm_cache)
        dXz, dWz, dbz = l.fc_backward(dhz, hz_cache)

        dX = dXr + dXz
        dh_old3 = dX[:, :self.H]

        dh_next = dh_old1 + dh_old2 + dh_old3

        grads = dict(Wz=dWz, Wr=dWr, Wh=dWh, Wy=dWy, bz=dbz, br=dbr, bh=dbh, by=dby)

        return dX, dh_next, grads

    def train_forward(self, X_train, h):
        ys, fc_caches, do_caches = [], [], []

        for X in X_train:
            X = X.reshape(1, -1) # X_1xn
            y, h, fc_cache = self.forward(X, h, self.model)
            y, do_cache = l.dropout_forward(y, self.p_dropout)
            fc_caches.append(fc_cache)
            do_caches.append(do_cache)
            ys.append(y)
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        caches = (fc_caches, do_caches)
        
        return ys, caches
                                
    def loss_function(self, y_pred, y_train): # , alpha alpha: learning rate
        loss, dys = 0.0, []

        for y, Y in zip(y_pred, y_train):
            loss += l2_regression(y_pred=y, y_train=Y)
            dy = dl2_regression(y_pred=y, y_train=Y)
            dys.append(dy)
            
        return loss, dys
    
    def train_backward(self, dys, caches):
        fc_caches, do_caches = caches
        
        dh = np.zeros((1, self.H)) 
        grad = {key: np.zeros_like(val) for key, val in self.model.items()}
        grads= {key: np.zeros_like(val) for key, val in self.model.items()}

        for t in reversed(range(len(dys))):
            dy = dys[t].reshape(1, -1) # dy_1xn
            dy = l.dropout_backward(dy, do_caches[t])
            dX, dh, grad = self.backward(dy, dh, fc_caches[t])
            for key in grad.keys():
                grads[key] += grad[key]
                
        return dX, grads
    
    def test(self, X_seed, h, size):
        ys = []
        X = X_seed.reshape(1, -1)
        for _ in range(size):
            y, h, _ = self.forward(X, h, self.model)
            X = y.copy() # previous out for the next input for prediction
            ys.append(y) # list array
        
        ys = np.array(ys, dtype=float).reshape(len(ys), -1) # ys_txn instead of ys_tx1xn
        return ys


    

In [8]:

    

def get_minibatch(X, y, minibatch_size, shuffle):
    minibatches = []

    # for i in range(0, X.shape[0] - minibatch_size + 1, 1):
    for i in range(0, X.shape[0], minibatch_size):
        X_mini = X[i:(i + minibatch_size)]
        y_mini = y[i:(i + minibatch_size)]
        minibatches.append((X_mini, y_mini))

    return minibatches

def adam_rnn(nn, XY_train, XY_valid, alpha, mb_size, n_iter, print_after):
    X_train, y_train = XY_train
    X_valid, y_valid = XY_valid

    # Momentum
    M= {key: np.zeros_like(val) for key, val in nn.model.items()}
    R= {key: np.zeros_like(val) for key, val in nn.model.items()}
    
    # Learning decay: suggested by Justin Jhonson in Standford
    beta1 = .9
    beta2 = .99
    state = nn.initial_state()
    smooth_loss = 1.
    minibatches = get_minibatch(X_train, y_train, mb_size, shuffle=False)
    
    # Epochs: iterating through the whole data
    for iter in range(1, n_iter + 1):
        
        # Minibatches
        for idx in range(len(minibatches)):
            
            # Train the model
            X_mini, y_mini = minibatches[idx]
            ys, caches = nn.train_forward(X_mini, state)
            loss, dys = nn.loss_function(y_pred=ys, y_train=y_mini) #, alpha=alpha
            _, grads = nn.train_backward(dys, caches)
            nn.losses['train'].append(loss)
            smooth_loss = (0.999 * smooth_loss) + (0.001 * loss)
            nn.losses['smooth train'].append(smooth_loss)
            
            # Update the model
            for key in grads.keys(): #key, value: items
                M[key] = l.exp_running_avg(M[key], grads[key], beta1)
                R[key] = l.exp_running_avg(R[key], grads[key]**2, beta2)
                m_k_hat = M[key] / (1. - (beta1** iter))
                r_k_hat = R[key] / (1. - (beta2** iter))
                nn.model[key] -= alpha * m_k_hat / (np.sqrt(r_k_hat) + l.eps)

            # Validate the model (by testing)
            ys = nn.test(X_seed=X_valid[0], h=state, size=X_valid.shape[0]) # ys_tx1xn
            valid_loss, _ = nn.loss_function(y_pred=ys, y_train=Y_valid) #, alpha=alpha
            nn.losses['valid'].append(valid_loss)

        # Print the model loss/ error
        if iter % print_after == 0:
            print('Iter-{}, train loss: {:.8f}, valid loss: {:.8f}'.format(iter, loss, valid_loss))

    return nn


    

In [ ]:

    

# Hyper-parameters
time_step = 64 # minibatch size: 32, 64, 128, or 256 Cache
n_iter = 1000 # epochs
alpha = 1e-4 # learning_rate: 1e-3, 5e-4, 1e-4 - default choices
print_after = 1 # print training loss, valid, and test
num_hidden_units = 64 # num_hidden_units in hidden layer
num_input_units = X_train.shape[1] # X_txn: noise given by using all possible channels/ features
keep_prob = 0.95 # p_dropout == keep_prob: keeping neurons/units function - default 0.95 to 0.9 based on SELU-Dropout

# Build the network and learning it or optimizing it using SGD
# def adam_rnn(nn, X_train, y_train, alpha=0.001, mb_size=256, n_iter=2000, print_after=100):
net = GRU(D=num_input_units, H=num_hidden_units, p_dropout=keep_prob) # , lam=lam, L=num_layers, p_dropout=p_dropout

# Start learning using BP-SGD-ADAM
adam_rnn(nn=net, XY_train=(X_train, Y_train), XY_valid=(X_valid, Y_valid), alpha=alpha, mb_size=time_step,
         n_iter=n_iter, print_after=print_after)


    

    




Iter-1, train loss: 30.70212305, valid loss: 632.91934660
Iter-2, train loss: 30.17579575, valid loss: 632.69786171
Iter-3, train loss: 30.08744097, valid loss: 635.27840650
Iter-4, train loss: 30.40181628, valid loss: 666.14573646
Iter-5, train loss: 30.00232400, valid loss: 752.34526715
Iter-6, train loss: 29.86732310, valid loss: 880.12281004
Iter-7, train loss: 28.30336806, valid loss: 832.42841871
Iter-8, train loss: 29.79148862, valid loss: 802.08941679
Iter-9, train loss: 29.84220890, valid loss: 868.66371282
Iter-10, train loss: 29.90412142, valid loss: 963.53426846
Iter-11, train loss: 30.71756957, valid loss: 916.34512439
Iter-12, train loss: 29.74213918, valid loss: 1004.43192182
Iter-13, train loss: 30.46093117, valid loss: 1400.09933846
Iter-14, train loss: 30.05437331, valid loss: 986.75113477
Iter-15, train loss: 30.58753627, valid loss: 1298.61669066
Iter-16, train loss: 30.55212958, valid loss: 1047.62877607
Iter-17, train loss: 30.71198905, valid loss: 1073.92001897
Iter-18, train loss: 27.04722485, valid loss: 1259.11482829
Iter-19, train loss: 27.64439766, valid loss: 1384.39735232
Iter-20, train loss: 30.19099158, valid loss: 1313.81906012
Iter-21, train loss: 30.28023966, valid loss: 1397.98143584
Iter-22, train loss: 27.03955134, valid loss: 1402.28432113
Iter-23, train loss: 30.85597289, valid loss: 1377.98352331
Iter-24, train loss: 29.20736875, valid loss: 1447.17812396
Iter-25, train loss: 29.47849138, valid loss: 1495.54829257
Iter-26, train loss: 30.83002910, valid loss: 1503.11051846
Iter-27, train loss: 30.85556952, valid loss: 1376.01707874
Iter-28, train loss: 30.03933915, valid loss: 1582.86834698
Iter-29, train loss: 30.99230841, valid loss: 1445.41011524
Iter-30, train loss: 30.78931562, valid loss: 1553.25684833
Iter-31, train loss: 27.73400154, valid loss: 1352.23250945
Iter-32, train loss: 30.22367727, valid loss: 1389.95785843
Iter-33, train loss: 30.49080710, valid loss: 1263.20661728
Iter-34, train loss: 30.51185627, valid loss: 1416.35684172
Iter-35, train loss: 30.78931562, valid loss: 1331.47127490
Iter-36, train loss: 29.39018576, valid loss: 1556.35928982
Iter-37, train loss: 30.35359908, valid loss: 1649.29295680
Iter-38, train loss: 30.37806550, valid loss: 1371.06230313
Iter-39, train loss: 30.98898092, valid loss: 1461.40429392
Iter-40, train loss: 30.49262781, valid loss: 1312.36686140
Iter-41, train loss: 29.77599164, valid loss: 1467.77781387
Iter-42, train loss: 30.90948185, valid loss: 1600.99555565
Iter-43, train loss: 29.69049801, valid loss: 1382.66469353
Iter-44, train loss: 30.24404075, valid loss: 1721.67101902
Iter-45, train loss: 30.78931562, valid loss: 1211.43233375
Iter-46, train loss: 29.09577580, valid loss: 1647.00053010
Iter-47, train loss: 30.67557173, valid loss: 1572.72723917
Iter-48, train loss: 29.15732748, valid loss: 1485.53673970
Iter-49, train loss: 27.17169690, valid loss: 1707.96359470
Iter-50, train loss: 30.07571038, valid loss: 1950.47841409
Iter-51, train loss: 30.63690026, valid loss: 1845.55250671
Iter-52, train loss: 30.35535057, valid loss: 1966.11775972
Iter-53, train loss: 28.91274802, valid loss: 1869.95262756
Iter-54, train loss: 30.36143782, valid loss: 1720.53456202
Iter-55, train loss: 28.74586188, valid loss: 1024.83505861
Iter-56, train loss: 30.35845877, valid loss: 1613.12492864
Iter-57, train loss: 30.01988351, valid loss: 1796.66602675
Iter-58, train loss: 29.56630914, valid loss: 1652.85446560
Iter-59, train loss: 30.02440820, valid loss: 2070.18010128
Iter-60, train loss: 30.43180045, valid loss: 1514.62953075
Iter-61, train loss: 29.72005838, valid loss: 979.83033161
Iter-62, train loss: 27.12031949, valid loss: 806.52461875
Iter-63, train loss: 30.34773904, valid loss: 782.67796312
Iter-64, train loss: 28.60385981, valid loss: 1085.53182475
Iter-65, train loss: 29.57069143, valid loss: 1570.66909564
Iter-66, train loss: 30.78931562, valid loss: 1353.19659806
Iter-67, train loss: 26.39224748, valid loss: 1462.36384689
Iter-68, train loss: 29.92786838, valid loss: 1822.82318965
Iter-69, train loss: 30.21619871, valid loss: 1744.61798495
Iter-70, train loss: 27.19515513, valid loss: 1395.95699605
Iter-71, train loss: 30.44694286, valid loss: 1590.26813876
Iter-72, train loss: 29.77158120, valid loss: 1531.76621024
Iter-73, train loss: 29.81913385, valid loss: 1523.82110131
Iter-74, train loss: 29.90050255, valid loss: 1384.02705336
Iter-75, train loss: 29.15700393, valid loss: 1652.87688627
Iter-76, train loss: 29.42857409, valid loss: 1385.46716332
Iter-77, train loss: 30.12563850, valid loss: 1180.03704078
Iter-78, train loss: 30.67215598, valid loss: 1535.00690282
Iter-79, train loss: 28.27658088, valid loss: 1555.98082134
Iter-80, train loss: 25.83162616, valid loss: 1148.58766767
Iter-81, train loss: 26.74605839, valid loss: 1072.82985254
Iter-82, train loss: 30.72264320, valid loss: 1105.35654369
Iter-83, train loss: 28.59825441, valid loss: 1459.91659342
Iter-84, train loss: 28.20706507, valid loss: 1184.10388833
Iter-85, train loss: 30.28211220, valid loss: 1146.22275891
Iter-86, train loss: 29.40363042, valid loss: 1655.14747577
Iter-87, train loss: 26.35950894, valid loss: 1178.22600320
Iter-88, train loss: 29.85916974, valid loss: 1340.72162524
Iter-89, train loss: 29.88020821, valid loss: 1502.62358711
Iter-90, train loss: 30.78931562, valid loss: 1279.93151143
Iter-91, train loss: 29.07609766, valid loss: 1354.70635334
Iter-92, train loss: 30.02183055, valid loss: 1693.31782813
Iter-93, train loss: 29.57807023, valid loss: 1400.82902306
Iter-94, train loss: 30.27647422, valid loss: 1640.14729746
Iter-95, train loss: 29.25902564, valid loss: 1503.01617695
Iter-96, train loss: 30.77083866, valid loss: 1443.87002864
Iter-97, train loss: 28.72671652, valid loss: 1265.14560972
Iter-98, train loss: 30.34708713, valid loss: 1153.09133366
Iter-99, train loss: 30.15931326, valid loss: 1334.10757864
Iter-100, train loss: 24.59819297, valid loss: 977.26264534
Iter-101, train loss: 30.78265636, valid loss: 1454.39630922
Iter-102, train loss: 29.36956504, valid loss: 1553.24515903
Iter-103, train loss: 30.18437578, valid loss: 1471.32606719
Iter-104, train loss: 29.24502470, valid loss: 1230.26325829
Iter-105, train loss: 30.50846303, valid loss: 1886.83813084
Iter-106, train loss: 30.15373529, valid loss: 1470.95590113
Iter-107, train loss: 30.38092129, valid loss: 1496.59993718
Iter-108, train loss: 30.30817498, valid loss: 1369.06160594
Iter-109, train loss: 30.31007859, valid loss: 1417.42383414
Iter-110, train loss: 28.84648645, valid loss: 1401.37654947
Iter-111, train loss: 27.15842715, valid loss: 942.82967707
Iter-112, train loss: 30.69808023, valid loss: 1475.68780036
Iter-113, train loss: 30.15931638, valid loss: 1364.99835619
Iter-114, train loss: 30.37891792, valid loss: 1700.12501080
Iter-115, train loss: 30.78931562, valid loss: 1573.29269564
Iter-116, train loss: 30.09084612, valid loss: 1368.54899427
Iter-117, train loss: 29.87617378, valid loss: 1189.30995539
Iter-118, train loss: 29.92358734, valid loss: 1408.80768986
Iter-119, train loss: 30.50504411, valid loss: 1854.97250165
Iter-120, train loss: 27.15715654, valid loss: 1713.25118024
Iter-121, train loss: 26.31996892, valid loss: 1366.88107749
Iter-122, train loss: 26.37970362, valid loss: 1544.40814746
Iter-123, train loss: 30.09071976, valid loss: 1426.54225219
Iter-124, train loss: 30.08276457, valid loss: 1888.22724651
Iter-125, train loss: 30.06129542, valid loss: 1520.86495066
Iter-126, train loss: 29.28638952, valid loss: 1452.70786639
Iter-127, train loss: 28.64661130, valid loss: 1187.41032025
Iter-128, train loss: 29.84242552, valid loss: 1800.30892503
Iter-129, train loss: 29.00596291, valid loss: 1532.33156837
Iter-130, train loss: 30.15822247, valid loss: 1449.03415649
Iter-131, train loss: 29.72724628, valid loss: 1659.53103071
Iter-132, train loss: 30.49351651, valid loss: 1919.26403104
Iter-133, train loss: 29.04450942, valid loss: 1341.96635734
Iter-134, train loss: 30.07220169, valid loss: 1641.19083879
Iter-135, train loss: 29.08456805, valid loss: 1721.54855329
Iter-136, train loss: 29.74785309, valid loss: 1005.26243146
Iter-137, train loss: 24.75319641, valid loss: 1122.20828309
Iter-138, train loss: 30.02162449, valid loss: 1538.86072327
Iter-139, train loss: 29.93958947, valid loss: 781.93737682
Iter-140, train loss: 30.50808574, valid loss: 1343.07667424
Iter-141, train loss: 29.53295214, valid loss: 1350.53491163
Iter-142, train loss: 28.40705856, valid loss: 1076.70177346
Iter-143, train loss: 25.65958682, valid loss: 1455.49841609
Iter-144, train loss: 30.06716674, valid loss: 987.56768717
Iter-145, train loss: 29.96990622, valid loss: 1012.88071788
Iter-146, train loss: 29.10914134, valid loss: 1303.12525153
Iter-147, train loss: 29.05136819, valid loss: 1256.58704500
Iter-148, train loss: 26.83495637, valid loss: 848.59868213
Iter-149, train loss: 28.57819678, valid loss: 1164.57081678
Iter-150, train loss: 29.67036948, valid loss: 1282.66753114
Iter-151, train loss: 30.21313948, valid loss: 968.56901497
Iter-152, train loss: 28.74430456, valid loss: 1244.70778137
Iter-153, train loss: 30.04886164, valid loss: 1043.42671808
Iter-154, train loss: 30.41289950, valid loss: 1685.80306940
Iter-155, train loss: 30.31416284, valid loss: 1179.93623598
Iter-156, train loss: 30.10152761, valid loss: 962.81327717
Iter-157, train loss: 30.05763376, valid loss: 1334.58112670
Iter-158, train loss: 30.30672804, valid loss: 1276.97877261
Iter-159, train loss: 27.92135975, valid loss: 1173.82613730
Iter-160, train loss: 30.80705745, valid loss: 1322.81689450
Iter-161, train loss: 29.55215834, valid loss: 1006.68152157
Iter-162, train loss: 30.21562309, valid loss: 830.51558123
Iter-163, train loss: 30.42332204, valid loss: 1546.46745616
Iter-164, train loss: 29.41996543, valid loss: 1003.67014789
Iter-165, train loss: 29.86930424, valid loss: 1497.46638522
Iter-166, train loss: 30.49373063, valid loss: 1106.92172112
Iter-167, train loss: 30.39001376, valid loss: 1064.73306101
Iter-168, train loss: 23.62553710, valid loss: 1125.66505094
Iter-169, train loss: 30.38568837, valid loss: 889.19918335
Iter-170, train loss: 27.76635712, valid loss: 822.34342091
Iter-171, train loss: 30.00719064, valid loss: 946.48128246
Iter-172, train loss: 30.56944302, valid loss: 980.34863458
Iter-173, train loss: 30.78931562, valid loss: 1350.44455161
Iter-174, train loss: 30.49120208, valid loss: 880.49916392
Iter-175, train loss: 27.95824845, valid loss: 1122.43810585
Iter-176, train loss: 29.21253183, valid loss: 1743.36330115
Iter-177, train loss: 30.61295675, valid loss: 1450.97662367
Iter-178, train loss: 30.08910600, valid loss: 946.47573141
Iter-179, train loss: 30.22864917, valid loss: 1137.39186086
Iter-180, train loss: 29.86614100, valid loss: 1336.55500791
Iter-181, train loss: 27.10501109, valid loss: 859.45027784
Iter-182, train loss: 28.91747888, valid loss: 1160.11932812
Iter-183, train loss: 28.29401535, valid loss: 1097.37147926
Iter-184, train loss: 29.24563137, valid loss: 1733.76193804
Iter-185, train loss: 29.64977950, valid loss: 1882.15166524
Iter-186, train loss: 29.79116718, valid loss: 1670.67116930
Iter-187, train loss: 29.30539686, valid loss: 767.48862312
Iter-188, train loss: 29.48696039, valid loss: 856.45809447
Iter-189, train loss: 29.35748134, valid loss: 850.70543816
Iter-190, train loss: 24.95918115, valid loss: 1473.26714293
Iter-191, train loss: 30.78931562, valid loss: 1313.14784772
Iter-192, train loss: 30.07088192, valid loss: 1067.27545062
Iter-193, train loss: 30.78931562, valid loss: 1062.37302208
Iter-194, train loss: 27.04316408, valid loss: 1459.16393778
Iter-195, train loss: 28.79485736, valid loss: 818.24349785
Iter-196, train loss: 29.48756395, valid loss: 855.70121044
Iter-197, train loss: 26.95310116, valid loss: 1497.32834768
Iter-198, train loss: 30.23165564, valid loss: 800.68753245
Iter-199, train loss: 29.71789152, valid loss: 1465.10779316
Iter-200, train loss: 30.77460468, valid loss: 1170.60641060
Iter-201, train loss: 26.84355237, valid loss: 915.94848834
Iter-202, train loss: 30.82910392, valid loss: 1068.69638501
Iter-203, train loss: 29.72738014, valid loss: 973.04833088
Iter-204, train loss: 27.05509548, valid loss: 1136.51119730
Iter-205, train loss: 29.78761764, valid loss: 1457.35338798
Iter-206, train loss: 30.27979821, valid loss: 1172.46422752
Iter-207, train loss: 29.57623330, valid loss: 1699.61229597
Iter-208, train loss: 29.46363359, valid loss: 942.09196255
Iter-209, train loss: 26.22064317, valid loss: 779.81623165
Iter-210, train loss: 28.20943995, valid loss: 1198.47833010
Iter-211, train loss: 29.74413520, valid loss: 1279.76348222
Iter-212, train loss: 29.99894263, valid loss: 1071.87405895
Iter-213, train loss: 29.41455337, valid loss: 1417.99345685
Iter-214, train loss: 28.51342271, valid loss: 1089.60571835
Iter-215, train loss: 29.18529777, valid loss: 1170.24754593
Iter-216, train loss: 30.36540532, valid loss: 1236.33850626
Iter-217, train loss: 30.78931562, valid loss: 759.97532812
Iter-218, train loss: 26.73483585, valid loss: 1209.81463909
Iter-219, train loss: 29.31262319, valid loss: 1084.15114562
Iter-220, train loss: 30.17629391, valid loss: 1228.11045612
Iter-221, train loss: 29.67994378, valid loss: 819.80115572
Iter-222, train loss: 30.47073237, valid loss: 1537.41268985
Iter-223, train loss: 30.78931562, valid loss: 1250.77897124
Iter-224, train loss: 30.25032561, valid loss: 1451.19906105
Iter-225, train loss: 30.08552280, valid loss: 762.88778776
Iter-226, train loss: 30.63606973, valid loss: 1149.86842155
Iter-227, train loss: 25.82134739, valid loss: 957.84891034
Iter-228, train loss: 30.78931562, valid loss: 2022.68259410
Iter-229, train loss: 30.30476499, valid loss: 631.98728584
Iter-230, train loss: 30.78931562, valid loss: 1085.27725338
Iter-231, train loss: 29.04979519, valid loss: 1020.46816153
Iter-232, train loss: 27.75726331, valid loss: 899.16327541
Iter-233, train loss: 28.99738338, valid loss: 1014.57122971
Iter-234, train loss: 30.75968998, valid loss: 1073.61759906
Iter-235, train loss: 27.67644536, valid loss: 976.95805233
Iter-236, train loss: 30.78624715, valid loss: 1130.24887905
Iter-237, train loss: 29.56112192, valid loss: 1703.20241782
Iter-238, train loss: 24.80795166, valid loss: 865.06642671
Iter-239, train loss: 26.98027915, valid loss: 1347.63885910
Iter-240, train loss: 29.92405713, valid loss: 1408.64127667
Iter-241, train loss: 28.88998066, valid loss: 1600.92518956
Iter-242, train loss: 30.73400037, valid loss: 614.38445799
Iter-243, train loss: 26.48109906, valid loss: 823.12486837
Iter-244, train loss: 28.39300975, valid loss: 695.51440103
Iter-245, train loss: 29.75623534, valid loss: 769.35292279
Iter-246, train loss: 29.96633268, valid loss: 669.89931900
Iter-247, train loss: 30.75450986, valid loss: 877.74903556
Iter-248, train loss: 29.44559091, valid loss: 865.39013897
Iter-249, train loss: 30.75052752, valid loss: 1566.08825858
Iter-250, train loss: 28.68262958, valid loss: 1207.93834672
Iter-251, train loss: 29.20091181, valid loss: 799.69035971
Iter-252, train loss: 29.76631217, valid loss: 799.12169667
Iter-253, train loss: 30.16126760, valid loss: 884.96828751
Iter-254, train loss: 30.52178061, valid loss: 1394.75034402
Iter-255, train loss: 28.10693847, valid loss: 901.05559687
Iter-256, train loss: 29.92444510, valid loss: 1396.27990827
Iter-257, train loss: 30.28133220, valid loss: 1079.21720253
Iter-258, train loss: 29.39355713, valid loss: 927.19665585
Iter-259, train loss: 30.57051057, valid loss: 1083.25631874
Iter-260, train loss: 29.52352214, valid loss: 856.58421194
Iter-261, train loss: 29.78600005, valid loss: 1114.86304763
Iter-262, train loss: 27.06143547, valid loss: 954.21824320
Iter-263, train loss: 28.62629693, valid loss: 1824.39231146
Iter-264, train loss: 27.49543352, valid loss: 1086.74561082
Iter-265, train loss: 29.76838268, valid loss: 1069.73937365
Iter-266, train loss: 29.59172373, valid loss: 913.13934582
Iter-267, train loss: 29.73773599, valid loss: 881.16476506
Iter-268, train loss: 29.73155107, valid loss: 1456.29994808
Iter-269, train loss: 30.39602912, valid loss: 744.10546078
Iter-270, train loss: 28.91947502, valid loss: 851.49987587
Iter-271, train loss: 29.28373972, valid loss: 1450.23683494
Iter-272, train loss: 30.81451404, valid loss: 1172.59054038
Iter-273, train loss: 26.09395886, valid loss: 880.12609105
Iter-274, train loss: 29.77014303, valid loss: 744.54688333
Iter-275, train loss: 28.41086052, valid loss: 1418.65588848
Iter-276, train loss: 28.40671026, valid loss: 1022.75565523
Iter-277, train loss: 30.19682612, valid loss: 822.73830613
Iter-278, train loss: 27.82315563, valid loss: 1121.51437013
Iter-279, train loss: 29.17329637, valid loss: 1216.60675998
Iter-280, train loss: 30.27974121, valid loss: 924.10978252
Iter-281, train loss: 30.37357494, valid loss: 871.05607936
Iter-282, train loss: 30.18428896, valid loss: 897.01280143
Iter-283, train loss: 30.30958609, valid loss: 838.06096674
Iter-284, train loss: 30.15239621, valid loss: 809.69915164
Iter-285, train loss: 29.59630148, valid loss: 1586.49301269
Iter-286, train loss: 30.53289899, valid loss: 882.21202786
Iter-287, train loss: 30.80174536, valid loss: 738.84760839
Iter-288, train loss: 30.13730363, valid loss: 996.05016607
Iter-289, train loss: 27.55453067, valid loss: 808.31581455
Iter-290, train loss: 30.06230087, valid loss: 801.91278834
Iter-291, train loss: 30.15235579, valid loss: 851.25191874
Iter-292, train loss: 27.70340210, valid loss: 975.63595682
Iter-293, train loss: 27.14097566, valid loss: 1608.37010041
Iter-294, train loss: 30.05538018, valid loss: 939.25628544
Iter-295, train loss: 29.37906657, valid loss: 1198.35582317
Iter-296, train loss: 29.78038331, valid loss: 912.64236604
Iter-297, train loss: 30.42563943, valid loss: 997.46602859
Iter-298, train loss: 30.33758201, valid loss: 916.35545192
Iter-299, train loss: 28.75131870, valid loss: 1079.02402115
Iter-300, train loss: 30.64558165, valid loss: 824.61962983
Iter-301, train loss: 29.03509961, valid loss: 1131.98614614
Iter-302, train loss: 28.10014619, valid loss: 1098.83294932
Iter-303, train loss: 29.98827270, valid loss: 963.41077701
Iter-304, train loss: 30.36583446, valid loss: 1160.91614114
Iter-305, train loss: 30.26675099, valid loss: 878.33854840
Iter-306, train loss: 30.20885094, valid loss: 917.82900363
Iter-307, train loss: 28.92749099, valid loss: 979.78254521
Iter-308, train loss: 25.89203937, valid loss: 876.85570914
Iter-309, train loss: 30.78931562, valid loss: 956.80053300
Iter-310, train loss: 29.94826980, valid loss: 1246.28175816
Iter-311, train loss: 29.90483523, valid loss: 1142.53291133
Iter-312, train loss: 30.47132037, valid loss: 813.79263848
Iter-313, train loss: 29.42031237, valid loss: 1161.57846299
Iter-314, train loss: 30.36127029, valid loss: 972.25059388
Iter-315, train loss: 29.33594723, valid loss: 1119.91916281
Iter-316, train loss: 24.99095133, valid loss: 922.35717679
Iter-317, train loss: 29.55795984, valid loss: 833.32238939
Iter-318, train loss: 30.76387816, valid loss: 792.26123794
Iter-319, train loss: 28.41579963, valid loss: 838.07298367
Iter-320, train loss: 28.83714492, valid loss: 983.33202660
Iter-321, train loss: 29.91316616, valid loss: 1051.79611426
Iter-322, train loss: 29.89547324, valid loss: 950.83889211
Iter-323, train loss: 30.74647325, valid loss: 1108.49692689
Iter-324, train loss: 29.58564819, valid loss: 1145.05387552
Iter-325, train loss: 29.38370799, valid loss: 984.64579375
Iter-326, train loss: 30.75599434, valid loss: 1105.68047673
Iter-327, train loss: 26.14788564, valid loss: 908.84301636
Iter-328, train loss: 30.03236333, valid loss: 1082.43783670
Iter-329, train loss: 30.23127369, valid loss: 952.52267076
Iter-330, train loss: 27.14288941, valid loss: 986.82370728
Iter-331, train loss: 30.59018603, valid loss: 1121.64992114
Iter-332, train loss: 30.78931562, valid loss: 762.71284500
Iter-333, train loss: 29.65406575, valid loss: 878.95749207
Iter-334, train loss: 26.96580313, valid loss: 1013.93400675
Iter-335, train loss: 30.00961360, valid loss: 1181.20081377
Iter-336, train loss: 30.15708952, valid loss: 948.88861571
Iter-337, train loss: 29.74568446, valid loss: 1096.80245188
Iter-338, train loss: 29.72220176, valid loss: 1083.80741761
Iter-339, train loss: 28.63454734, valid loss: 864.20509922
Iter-340, train loss: 30.37644317, valid loss: 991.32733980
Iter-341, train loss: 27.20886584, valid loss: 824.21327791
Iter-342, train loss: 30.69325008, valid loss: 922.87786684
Iter-343, train loss: 29.68781720, valid loss: 1442.41790666
Iter-344, train loss: 29.77164938, valid loss: 1016.52302627
Iter-345, train loss: 30.32160906, valid loss: 922.64061971
Iter-346, train loss: 29.93100007, valid loss: 998.95000058
Iter-347, train loss: 30.30466092, valid loss: 874.92735195
Iter-348, train loss: 26.00413059, valid loss: 912.17941384
Iter-349, train loss: 26.42050397, valid loss: 943.87071933
Iter-350, train loss: 25.59960824, valid loss: 1240.40314811
Iter-351, train loss: 29.86140338, valid loss: 906.76728254
Iter-352, train loss: 28.59621436, valid loss: 939.26560267
Iter-353, train loss: 30.27486606, valid loss: 974.92217822
Iter-354, train loss: 27.22282086, valid loss: 834.03412858
Iter-355, train loss: 29.74229780, valid loss: 777.30840294
Iter-356, train loss: 30.78931562, valid loss: 937.05207841
Iter-357, train loss: 28.83931484, valid loss: 910.96592068
Iter-358, train loss: 30.35926446, valid loss: 921.27799852
Iter-359, train loss: 29.84589679, valid loss: 706.44544095
Iter-360, train loss: 30.15312927, valid loss: 979.57301457
Iter-361, train loss: 29.94991150, valid loss: 886.95182258
Iter-362, train loss: 29.48367201, valid loss: 927.77858874
Iter-363, train loss: 30.60297181, valid loss: 1019.16988366
Iter-364, train loss: 28.50547575, valid loss: 1146.41952077
Iter-365, train loss: 30.20747430, valid loss: 891.77285313
Iter-366, train loss: 27.91255226, valid loss: 929.88144869
Iter-367, train loss: 30.75871348, valid loss: 1150.37019176
Iter-368, train loss: 29.33936449, valid loss: 1038.66508232
Iter-369, train loss: 25.47018111, valid loss: 1323.80197648
Iter-370, train loss: 30.13618496, valid loss: 980.38457330
Iter-371, train loss: 30.14195649, valid loss: 1033.06443970
Iter-372, train loss: 30.55149908, valid loss: 1115.53014080
Iter-373, train loss: 30.76873387, valid loss: 1190.82974364
Iter-374, train loss: 28.96405740, valid loss: 1141.25338504
Iter-375, train loss: 29.76336384, valid loss: 1094.45726665
Iter-376, train loss: 29.91232505, valid loss: 1030.46760962
Iter-377, train loss: 26.28712158, valid loss: 875.35786784
Iter-378, train loss: 30.10733069, valid loss: 927.47081633
Iter-379, train loss: 28.02974892, valid loss: 1290.74613479
Iter-380, train loss: 30.26136937, valid loss: 850.31182704
Iter-381, train loss: 28.66301576, valid loss: 979.77571071
Iter-382, train loss: 29.93518966, valid loss: 883.34640982
Iter-383, train loss: 27.55760357, valid loss: 1168.00911688
Iter-384, train loss: 30.70568219, valid loss: 1164.74069718
Iter-385, train loss: 29.81657797, valid loss: 953.05409128
Iter-386, train loss: 26.50105121, valid loss: 1099.00265666
Iter-387, train loss: 27.64978329, valid loss: 920.42493869
Iter-388, train loss: 30.20857944, valid loss: 1023.22700903
Iter-389, train loss: 30.61367967, valid loss: 1258.23667302
Iter-390, train loss: 30.40375528, valid loss: 1040.37513514
Iter-391, train loss: 30.75376657, valid loss: 1293.18851526
Iter-392, train loss: 29.15909109, valid loss: 1006.35002085
Iter-393, train loss: 30.77874396, valid loss: 1231.69606228
Iter-394, train loss: 29.25193189, valid loss: 442.70771593
Iter-395, train loss: 30.09267280, valid loss: 1049.68577988
Iter-396, train loss: 28.21947324, valid loss: 731.39775311
Iter-397, train loss: 30.57848041, valid loss: 1396.29578531
Iter-398, train loss: 30.33738558, valid loss: 1409.98656073
Iter-399, train loss: 29.27099100, valid loss: 794.59784964
Iter-400, train loss: 25.65056665, valid loss: 982.55649783
Iter-401, train loss: 29.80894674, valid loss: 1110.28966516
Iter-402, train loss: 30.17268737, valid loss: 1127.34951914
Iter-403, train loss: 26.74736964, valid loss: 1774.31262416
Iter-404, train loss: 29.53331232, valid loss: 1042.81464267
Iter-405, train loss: 30.50512848, valid loss: 966.86815121
Iter-406, train loss: 30.49732482, valid loss: 939.47493850
Iter-407, train loss: 29.85472885, valid loss: 1165.84541916
Iter-408, train loss: 29.46986367, valid loss: 1183.68873611
Iter-409, train loss: 30.29988396, valid loss: 1100.56300459
Iter-410, train loss: 30.49830499, valid loss: 1270.10924953
Iter-411, train loss: 27.27679173, valid loss: 1028.28429480
Iter-412, train loss: 29.05101358, valid loss: 995.92846936
Iter-413, train loss: 28.92701101, valid loss: 1054.85973365
Iter-414, train loss: 30.40566469, valid loss: 1165.52742228
Iter-415, train loss: 30.72152084, valid loss: 987.20334111
Iter-416, train loss: 30.78931562, valid loss: 1381.27468063
Iter-417, train loss: 30.27864463, valid loss: 1092.99080676
Iter-418, train loss: 28.13420150, valid loss: 1184.97114748
Iter-419, train loss: 28.95174598, valid loss: 1153.91504394
Iter-420, train loss: 30.18638923, valid loss: 1094.80383998
Iter-421, train loss: 30.07481692, valid loss: 1068.75150177
Iter-422, train loss: 28.89286424, valid loss: 1217.72229614
Iter-423, train loss: 29.54142814, valid loss: 914.32344869
Iter-424, train loss: 29.60236539, valid loss: 808.23949761
Iter-425, train loss: 29.58899892, valid loss: 835.96102921
Iter-426, train loss: 30.53270148, valid loss: 912.12962626
Iter-427, train loss: 29.95460766, valid loss: 945.62643954
Iter-428, train loss: 29.99274777, valid loss: 1181.05168802
Iter-429, train loss: 30.78931562, valid loss: 1045.42054007
Iter-430, train loss: 27.45093970, valid loss: 944.06401854
Iter-431, train loss: 30.24932815, valid loss: 961.21796759
Iter-432, train loss: 29.63613248, valid loss: 989.40201405
Iter-433, train loss: 27.31092024, valid loss: 1073.17496896
Iter-434, train loss: 30.64186222, valid loss: 1186.54182148
Iter-435, train loss: 27.03276728, valid loss: 955.66126211
Iter-436, train loss: 27.64409323, valid loss: 1061.65450579
Iter-437, train loss: 30.01070191, valid loss: 843.47433088
Iter-438, train loss: 29.81471547, valid loss: 823.20533795
Iter-439, train loss: 29.31152931, valid loss: 985.40355180
Iter-440, train loss: 30.65269181, valid loss: 982.79884538
Iter-441, train loss: 29.83792455, valid loss: 974.43024717
Iter-442, train loss: 30.39671599, valid loss: 984.09230746
Iter-443, train loss: 30.53778121, valid loss: 1286.71207109
Iter-444, train loss: 27.19366688, valid loss: 1061.89188614
Iter-445, train loss: 30.53064322, valid loss: 767.79732315
Iter-446, train loss: 29.59482540, valid loss: 1143.34311686
Iter-447, train loss: 27.64823443, valid loss: 1314.92055797
Iter-448, train loss: 30.50641533, valid loss: 1006.67137408
Iter-449, train loss: 29.89315966, valid loss: 851.57585015
Iter-450, train loss: 29.83509926, valid loss: 979.14234518
Iter-451, train loss: 29.80260001, valid loss: 1163.87376183
Iter-452, train loss: 26.98802837, valid loss: 1064.67279451
Iter-453, train loss: 29.50154671, valid loss: 1195.23617658
Iter-454, train loss: 30.55388021, valid loss: 1289.30352323
Iter-455, train loss: 26.87333740, valid loss: 889.74563779
Iter-456, train loss: 26.61762779, valid loss: 995.80118542
Iter-457, train loss: 27.68267157, valid loss: 1223.95160916
Iter-458, train loss: 25.66935272, valid loss: 1024.92523089
Iter-459, train loss: 27.03378677, valid loss: 818.06343984
Iter-460, train loss: 30.35807789, valid loss: 970.13322958
Iter-461, train loss: 28.43695197, valid loss: 993.56183047
Iter-462, train loss: 28.72723287, valid loss: 995.92082317
Iter-463, train loss: 26.67298109, valid loss: 1039.43704751
Iter-464, train loss: 29.73084786, valid loss: 1148.29925714
Iter-465, train loss: 30.21721298, valid loss: 1085.93005925
Iter-466, train loss: 30.22452849, valid loss: 1036.66359238
Iter-467, train loss: 29.99789134, valid loss: 1080.32446417
Iter-468, train loss: 30.18583457, valid loss: 1130.49992301
Iter-469, train loss: 30.45114378, valid loss: 855.15725890
Iter-470, train loss: 29.63438462, valid loss: 1109.03332860
Iter-471, train loss: 30.50617984, valid loss: 1010.03628420
Iter-472, train loss: 30.48646382, valid loss: 1105.39671402
Iter-473, train loss: 28.84353036, valid loss: 928.26097032
Iter-474, train loss: 28.82336236, valid loss: 983.28493590
Iter-475, train loss: 28.70631563, valid loss: 935.03494752
Iter-476, train loss: 29.68945953, valid loss: 1215.07383890
Iter-477, train loss: 29.65262041, valid loss: 1042.93697655
Iter-478, train loss: 29.50632060, valid loss: 1012.41454160
Iter-479, train loss: 29.89475151, valid loss: 1293.12585504
Iter-480, train loss: 29.61191314, valid loss: 930.03154305
Iter-481, train loss: 29.82424094, valid loss: 1011.63663994
Iter-482, train loss: 29.41560156, valid loss: 984.65602348
Iter-483, train loss: 28.82333222, valid loss: 1040.04449331
Iter-484, train loss: 30.24614615, valid loss: 1045.96157391
Iter-485, train loss: 30.22615866, valid loss: 966.32852621
Iter-486, train loss: 30.79631639, valid loss: 886.76351576
Iter-487, train loss: 30.35885269, valid loss: 1065.31368756
Iter-488, train loss: 29.84014023, valid loss: 1101.19231936
Iter-489, train loss: 29.85587661, valid loss: 956.44602008
Iter-490, train loss: 30.62323577, valid loss: 1112.64572621
Iter-491, train loss: 29.29008888, valid loss: 1023.44673902
Iter-492, train loss: 29.62719843, valid loss: 1201.99557770
Iter-493, train loss: 29.44175173, valid loss: 1049.76494964
Iter-494, train loss: 29.22755293, valid loss: 893.03182818
Iter-495, train loss: 26.56413046, valid loss: 1003.41368576
Iter-496, train loss: 25.83116218, valid loss: 1161.82888576
Iter-497, train loss: 28.02404440, valid loss: 858.38355649
Iter-498, train loss: 30.58243896, valid loss: 1149.03198975
Iter-499, train loss: 28.66770933, valid loss: 840.51744414
Iter-500, train loss: 28.79766914, valid loss: 1214.02076422
Iter-501, train loss: 30.77517481, valid loss: 1235.97479620
Iter-502, train loss: 30.78331175, valid loss: 1057.71679304
Iter-503, train loss: 29.52480179, valid loss: 1245.62030717
Iter-504, train loss: 30.76558131, valid loss: 1184.98754670
Iter-505, train loss: 29.57643249, valid loss: 1073.85812233
Iter-506, train loss: 30.37222176, valid loss: 1031.70812039
Iter-507, train loss: 28.00439197, valid loss: 1223.38365090
Iter-508, train loss: 28.92552219, valid loss: 1086.13521121
Iter-509, train loss: 30.72321937, valid loss: 819.88042786
Iter-510, train loss: 30.09743169, valid loss: 1074.74403993
Iter-511, train loss: 29.14874104, valid loss: 1153.11796602
Iter-512, train loss: 27.36417604, valid loss: 1076.32645078
Iter-513, train loss: 28.77159743, valid loss: 994.12982985
Iter-514, train loss: 28.87997237, valid loss: 855.43301356
Iter-515, train loss: 30.06511094, valid loss: 1092.31012516
Iter-516, train loss: 29.66827810, valid loss: 925.58826355
Iter-517, train loss: 30.48686719, valid loss: 1066.30881349
Iter-518, train loss: 30.62343021, valid loss: 946.72309581
Iter-519, train loss: 30.24910613, valid loss: 1422.94760435
Iter-520, train loss: 27.29890717, valid loss: 1060.14736735
Iter-521, train loss: 30.48836343, valid loss: 957.88224976
Iter-522, train loss: 27.94647016, valid loss: 987.89930085
Iter-523, train loss: 27.07506038, valid loss: 1202.60229195
Iter-524, train loss: 29.55037729, valid loss: 1310.99145102
Iter-525, train loss: 29.39636166, valid loss: 1188.70260374
Iter-526, train loss: 29.84176124, valid loss: 1242.24720774
Iter-527, train loss: 29.83536888, valid loss: 1116.28614141
Iter-528, train loss: 30.69978683, valid loss: 981.09663618
Iter-529, train loss: 27.14218915, valid loss: 1198.52620743
Iter-530, train loss: 28.20396947, valid loss: 1120.75218391
Iter-531, train loss: 30.08648512, valid loss: 1193.25411542
Iter-532, train loss: 29.97034708, valid loss: 979.08952215
Iter-533, train loss: 30.29902351, valid loss: 1037.57623187
Iter-534, train loss: 30.75251639, valid loss: 1485.79041624
Iter-535, train loss: 30.19723058, valid loss: 1107.87822784
Iter-536, train loss: 28.73589729, valid loss: 1270.18306290
Iter-537, train loss: 29.08401672, valid loss: 1112.99956169
Iter-538, train loss: 30.33329025, valid loss: 1315.40338932
Iter-539, train loss: 30.01423685, valid loss: 960.21414372
Iter-540, train loss: 27.84270395, valid loss: 1069.72280644
Iter-541, train loss: 26.08371928, valid loss: 1136.83031729
Iter-542, train loss: 30.35801585, valid loss: 1161.30346470
Iter-543, train loss: 26.32193827, valid loss: 982.46258508
Iter-544, train loss: 29.58824434, valid loss: 1019.86282748
Iter-545, train loss: 27.23000038, valid loss: 1062.26038471
Iter-546, train loss: 30.31261561, valid loss: 959.56606623
Iter-547, train loss: 28.66922265, valid loss: 962.27207076
Iter-548, train loss: 30.78931562, valid loss: 999.07035519
Iter-549, train loss: 30.12270801, valid loss: 1354.65105839
Iter-550, train loss: 28.90157437, valid loss: 1075.10260622
Iter-551, train loss: 30.38966265, valid loss: 1184.52122186
Iter-552, train loss: 30.30422556, valid loss: 1147.32425549
Iter-553, train loss: 29.96884148, valid loss: 1148.42721888
Iter-554, train loss: 30.35800940, valid loss: 1147.49212739
Iter-555, train loss: 30.11783428, valid loss: 1215.14435901
Iter-556, train loss: 29.76509881, valid loss: 1029.55655300
Iter-557, train loss: 29.71808082, valid loss: 1096.11049817
Iter-558, train loss: 30.78931562, valid loss: 1010.40617781
Iter-559, train loss: 29.12028766, valid loss: 1086.60493282
Iter-560, train loss: 29.54641817, valid loss: 1035.20764706
Iter-561, train loss: 27.72924630, valid loss: 1129.94545306
Iter-562, train loss: 26.12788818, valid loss: 1034.39052568
Iter-563, train loss: 27.57023862, valid loss: 1140.84636917
Iter-564, train loss: 29.65131149, valid loss: 1175.43119160
Iter-565, train loss: 25.22034387, valid loss: 981.24179754
Iter-566, train loss: 30.50297407, valid loss: 1128.37929987
Iter-567, train loss: 28.29214975, valid loss: 1068.55680026
Iter-568, train loss: 29.52436080, valid loss: 1375.22619775
Iter-569, train loss: 29.70736786, valid loss: 1043.60465760
Iter-570, train loss: 29.38087655, valid loss: 924.54380400
Iter-571, train loss: 30.43141897, valid loss: 1110.57389072
Iter-572, train loss: 28.35443271, valid loss: 1081.97139594
Iter-573, train loss: 30.20991145, valid loss: 1068.19811718
Iter-574, train loss: 24.43696789, valid loss: 940.79016980
Iter-575, train loss: 29.51537254, valid loss: 822.79069486
Iter-576, train loss: 30.36624744, valid loss: 1005.53499945
Iter-577, train loss: 28.56360335, valid loss: 962.26609628
Iter-578, train loss: 30.68537337, valid loss: 945.40954387
Iter-579, train loss: 29.02949551, valid loss: 1100.26557525
Iter-580, train loss: 27.88832695, valid loss: 1243.48417395
Iter-581, train loss: 28.46898637, valid loss: 1015.19835705
Iter-582, train loss: 25.09513830, valid loss: 1056.55024561
Iter-583, train loss: 29.48718948, valid loss: 1044.78630443
Iter-584, train loss: 29.63812233, valid loss: 826.19380251
Iter-585, train loss: 30.01690104, valid loss: 1064.62363904
Iter-586, train loss: 29.72362554, valid loss: 1031.91255077
Iter-587, train loss: 29.67974181, valid loss: 1284.10793456
Iter-588, train loss: 30.21809289, valid loss: 1095.73800116
Iter-589, train loss: 29.36160962, valid loss: 1202.53304363
Iter-590, train loss: 29.59992538, valid loss: 1165.64733200
Iter-591, train loss: 28.41061838, valid loss: 1266.40311020
Iter-592, train loss: 29.00593208, valid loss: 1055.12591187
Iter-593, train loss: 22.70439948, valid loss: 1060.09027046
Iter-594, train loss: 30.09290565, valid loss: 1519.86703650
Iter-595, train loss: 30.19819165, valid loss: 1047.80082536
Iter-596, train loss: 29.88980208, valid loss: 950.04253408
Iter-597, train loss: 30.50817917, valid loss: 967.31718819
Iter-598, train loss: 30.62961467, valid loss: 1108.51092675
Iter-599, train loss: 29.91379524, valid loss: 1474.40184988
Iter-600, train loss: 26.73392188, valid loss: 1119.39697508
Iter-601, train loss: 30.30809977, valid loss: 1321.56717484
Iter-602, train loss: 30.58750590, valid loss: 1198.46263314
Iter-603, train loss: 30.65333940, valid loss: 975.51780040
Iter-604, train loss: 30.35877250, valid loss: 1158.82380870
Iter-605, train loss: 29.17168309, valid loss: 1354.73109737
Iter-606, train loss: 29.01164625, valid loss: 1528.52907412
Iter-607, train loss: 23.68896296, valid loss: 1094.69316801
Iter-608, train loss: 29.54994897, valid loss: 1354.05518481
Iter-609, train loss: 30.77797906, valid loss: 1067.97834500
Iter-610, train loss: 30.13991636, valid loss: 1225.13574331
Iter-611, train loss: 28.22969800, valid loss: 1027.55680211
Iter-612, train loss: 28.72090426, valid loss: 1011.38323867
Iter-613, train loss: 26.03832859, valid loss: 1251.61199326
Iter-614, train loss: 27.11702160, valid loss: 1102.21591384
Iter-615, train loss: 29.25840845, valid loss: 1158.48714619
Iter-616, train loss: 29.98962507, valid loss: 1241.98121594
Iter-617, train loss: 30.56736388, valid loss: 1219.55345356
Iter-618, train loss: 30.48345234, valid loss: 1059.83636594
Iter-619, train loss: 29.95148154, valid loss: 1266.98334276
Iter-620, train loss: 30.47856724, valid loss: 1085.00830820
Iter-621, train loss: 30.70001396, valid loss: 1170.44608050
Iter-622, train loss: 29.78041900, valid loss: 1164.89823977
Iter-623, train loss: 29.65622256, valid loss: 1152.22495900
Iter-624, train loss: 28.84254154, valid loss: 1646.07842348
Iter-625, train loss: 30.17327599, valid loss: 1077.40160550
Iter-626, train loss: 29.71843444, valid loss: 1267.64573147
Iter-627, train loss: 30.06193147, valid loss: 1158.46157974
Iter-628, train loss: 30.19020221, valid loss: 1090.76057833
Iter-629, train loss: 30.22297512, valid loss: 1108.30979285
Iter-630, train loss: 30.51790652, valid loss: 1519.38688117
Iter-631, train loss: 26.74825783, valid loss: 1035.97843180
Iter-632, train loss: 25.23667041, valid loss: 1266.45492906
Iter-633, train loss: 26.69794678, valid loss: 1182.40115596
Iter-634, train loss: 30.91336525, valid loss: 1102.54558295
Iter-635, train loss: 30.78931562, valid loss: 986.17754646
Iter-636, train loss: 30.21372101, valid loss: 987.03902054
Iter-637, train loss: 29.21792652, valid loss: 1060.16905270
Iter-638, train loss: 26.70806923, valid loss: 1297.98382650
Iter-639, train loss: 28.79411984, valid loss: 986.79081863
Iter-640, train loss: 29.37630936, valid loss: 1278.96383712
Iter-641, train loss: 29.09263672, valid loss: 934.08977205
Iter-642, train loss: 28.26617819, valid loss: 1171.82638149
Iter-643, train loss: 29.38960102, valid loss: 1162.32732893

In [ ]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['train'], label='Train loss')
plt.plot(net.losses['smooth train'], label='Train smooth loss')
plt.legend()
plt.show()


    

In [ ]:

    

import matplotlib.pyplot as plt

plt.plot(net.losses['valid'], label='Validation loss')
plt.legend()
plt.show()


    

In [ ]:

    

import matplotlib.pyplot as plt

y_pred = net.test(X_seed=X_valid[0], h=net.initial_state(), size=X_valid.shape[0]) # ys_tx1xn
y_pred.shape, Y_valid.shape

plt.plot(y_pred[:200, :], label='y_pred')
plt.plot(Y_valid[:200, :], label='Y_valid')
# plt.plot(X_valid[:100], label='X_valid')
plt.legend()
plt.show()


    

In [ ]:

    




    

In [ ]: