In [1]:
import numpy as np
import pandas as pd
%matplotlib inline
import matplotlib.pyplot as plt
In [2]:
a = np.array([1, 3, 2, 4])
In [3]:
a
Out[3]:
In [4]:
type(a)
Out[4]:
In [5]:
A = np.array([[3, 1, 2],
[2, 3, 4]])
B = np.array([[0, 1],
[2, 3],
[4, 5]])
C = np.array([[0, 1],
[2, 3],
[4, 5],
[0, 1],
[2, 3],
[4, 5]])
print("A is a {} matrix".format(A.shape))
print("B is a {} matrix".format(B.shape))
print("C is a {} matrix".format(C.shape))
In [6]:
A[0]
Out[6]:
In [7]:
C[2, 0]
Out[7]:
In [8]:
B[:, 0]
Out[8]:
In [9]:
3 * A
Out[9]:
In [10]:
A + A
Out[10]:
In [11]:
A * A
Out[11]:
In [12]:
A / A
Out[12]:
In [13]:
A - A
Out[13]:
In [14]:
# A + B will not work, because A and B need to have same dimensionality!
In [15]:
# A * B (element-wise mutliplication) will not work, because A and B need to have same dimensionality!
In [16]:
A.shape
Out[16]:
In [17]:
B.shape
Out[17]:
In [18]:
A.dot(B)
Out[18]:
In [19]:
np.dot(A, B)
Out[19]:
In [20]:
B.dot(A)
Out[20]:
In [21]:
C.shape
Out[21]:
In [22]:
A.shape
Out[22]:
In [23]:
C.dot(A)
Out[23]:
In [24]:
# A.dot(C) will not work, because shapes (2,3) and (6,2) are not aligned: 3 (dim 1) != 6 (dim 0)
In [25]:
df = pd.read_csv('./data/banknotes.csv')
In [26]:
df.head()
Out[26]:
In [27]:
df['class'].value_counts()
Out[27]:
In [28]:
import notebook
import seaborn as sns
In [29]:
sns.pairplot(df, hue = "class")
Out[29]:
In [30]:
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import scale
In [31]:
X = scale(df.drop('class', axis = 1).values)
y = df['class'].values
In [32]:
model = RandomForestClassifier()
cross_val_score(model, X, y)
Out[32]:
In [33]:
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size = 0.3,
random_state = 42)
In [34]:
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.optimizers import SGD
In [35]:
K.clear_session()
model = Sequential()
model.add(Dense(1,
input_shape = (4,),
activation = 'sigmoid'))
model.compile(loss = 'binary_crossentropy',
optimizer = 'sgd',
metrics = ['accuracy'])
history = model.fit(X_train, y_train)
result = model.evaluate(X_test, y_test)
In [36]:
historydf = pd.DataFrame(history.history,
index = history.epoch)
In [37]:
historydf.plot(figsize = (20, 10),
ylim = (0,1))
plt.title("Test accuracy: {:3.2f} %".format(result[1]*100), fontsize = 15)
Out[37]:
In [38]:
dflist = []
learning_rates = [0.01, 0.05, 0.1, 0.5]
for lr in learning_rates:
K.clear_session()
model = Sequential()
model.add(Dense(1,
input_shape = (4,),
activation = 'sigmoid'))
model.compile(loss = 'binary_crossentropy',
optimizer = SGD(lr = lr),
metrics = ['accuracy'])
h = model.fit(X_train,
y_train,
batch_size = 16,
verbose = 0)
dflist.append(pd.DataFrame(h.history,
index = h.epoch))
In [39]:
historydf = pd.concat(dflist, axis = 1)
In [40]:
historydf
Out[40]:
In [41]:
metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product([learning_rates, metrics_reported],
names = ['learning_rate', 'metric'])
historydf.columns = idx
In [42]:
historydf
Out[42]:
In [43]:
fig, ax = plt.subplots(figsize=(15, 10))
ax = plt.subplot(211)
historydf.xs('loss',
axis = 1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Loss")
ax = plt.subplot(212)
historydf.xs('acc',
axis = 1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Accuracy")
plt.xlabel("Epochs")
plt.tight_layout()
In [44]:
dflist = []
batch_sizes = [16, 32, 64, 128]
for batch_size in batch_sizes:
K.clear_session()
model = Sequential()
model.add(Dense(1,
input_shape = (4,),
activation = 'sigmoid'))
model.compile(loss = 'binary_crossentropy',
optimizer = 'sgd',
metrics = ['accuracy'])
h = model.fit(X_train,
y_train,
batch_size = batch_size,
verbose = 0)
dflist.append(pd.DataFrame(h.history,
index = h.epoch))
In [45]:
historydf = pd.concat(dflist,
axis = 1)
metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product([batch_sizes, metrics_reported],
names = ['batch_size', 'metric'])
historydf.columns = idx
In [46]:
historydf
Out[46]:
In [47]:
fig, ax = plt.subplots(figsize=(15, 10))
ax = plt.subplot(211)
historydf.xs('loss',
axis = 1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Loss")
ax = plt.subplot(212)
historydf.xs('acc',
axis=1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Accuracy")
plt.xlabel("Epochs")
plt.tight_layout()
In [48]:
from keras.optimizers import SGD, Adam, Adagrad, RMSprop
In [49]:
dflist = []
optimizers = ['SGD(lr = 0.01)',
'SGD(lr = 0.01, momentum = 0.3)',
'SGD(lr = 0.01, momentum = 0.3, nesterov = True)',
'Adam(lr = 0.01)',
'Adagrad(lr = 0.01)',
'RMSprop(lr = 0.01)']
for opt_name in optimizers:
K.clear_session()
model = Sequential()
model.add(Dense(1,
input_shape = (4,),
activation = 'sigmoid'))
model.compile(loss = 'binary_crossentropy',
optimizer = eval(opt_name),
metrics = ['accuracy'])
h = model.fit(X_train, y_train,
batch_size = 16,
epochs = 5,
verbose = 0)
dflist.append(pd.DataFrame(h.history,
index = h.epoch))
In [50]:
historydf = pd.concat(dflist,
axis = 1)
metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product([optimizers, metrics_reported],
names = ['optimizers', 'metric'])
historydf.columns = idx
In [51]:
fig, ax = plt.subplots(figsize=(15, 10))
ax = plt.subplot(211)
historydf.xs('loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Loss")
ax = plt.subplot(212)
historydf.xs('acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Accuracy")
plt.xlabel("Epochs")
plt.tight_layout()
In [52]:
dflist = []
initializers = ['zeros',
'uniform',
'normal',
'he_normal',
'lecun_uniform']
for init in initializers:
K.clear_session()
model = Sequential()
model.add(Dense(1,
input_shape = (4,),
kernel_initializer = init,
activation = 'sigmoid'))
model.compile(loss = 'binary_crossentropy',
optimizer = 'rmsprop',
metrics = ['accuracy'])
h = model.fit(X_train, y_train,
batch_size = 16,
epochs = 5,
verbose = 0)
dflist.append(pd.DataFrame(h.history, index=h.epoch))
In [53]:
historydf = pd.concat(dflist,
axis = 1)
metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product([initializers, metrics_reported],
names = ['initializers', 'metric'])
historydf.columns = idx
In [54]:
fig, ax = plt.subplots(figsize=(15, 10))
ax = plt.subplot(211)
historydf.xs('loss', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Loss")
ax = plt.subplot(212)
historydf.xs('acc', axis=1, level='metric').plot(ylim=(0,1), ax=ax)
plt.title("Accuracy")
plt.xlabel("Epochs")
plt.tight_layout()
In [55]:
K.clear_session()
model = Sequential()
model.add(Dense(2,
input_shape = (4,),
activation = 'relu'))
model.add(Dense(1,
activation = 'sigmoid'))
model.compile(loss = 'binary_crossentropy',
optimizer = RMSprop(lr = 0.01),
metrics = ['accuracy'])
h = model.fit(X_train, y_train,
batch_size = 16,
epochs = 20,
verbose = 1,
validation_split = 0.3)
result = model.evaluate(X_test, y_test)
In [56]:
result
Out[56]:
In [57]:
model.summary()
In [58]:
model.layers
Out[58]:
In [59]:
inp = model.layers[0].input
out = model.layers[0].output
In [60]:
inp
Out[60]:
In [61]:
out
Out[61]:
In [62]:
features_function = K.function([inp], [out])
In [63]:
features_function
Out[63]:
In [64]:
features_function([X_test])[0].shape
Out[64]:
In [65]:
features = features_function([X_test])[0]
In [66]:
plt.scatter(features[:, 0], features[:, 1], c = y_test, cmap='coolwarm')
Out[66]:
In [67]:
K.clear_session()
model = Sequential()
model.add(Dense(3,
input_shape = (4,),
activation = 'relu'))
model.add(Dense(2,
activation = 'relu'))
model.add(Dense(1,
activation = 'sigmoid'))
model.compile(loss = 'binary_crossentropy',
optimizer = RMSprop(lr=0.01),
metrics = ['accuracy'])
In [68]:
inp = model.layers[0].input
out = model.layers[1].output
features_function = K.function([inp], [out])
plt.figure(figsize = (15,10))
for i in range(1, 26):
plt.subplot(5, 5, i)
h = model.fit(X_train, y_train,
batch_size = 16,
epochs = 1,
verbose = 0)
test_accuracy = model.evaluate(X_test, y_test)[1]
features = features_function([X_test])[0]
plt.scatter(features[:, 0],
features[:, 1],
c = y_test,
cmap = 'coolwarm')
plt.xlim(-0.5, 3.5)
plt.ylim(-0.5, 4.0)
plt.title('Epoch: {}, Test Acc: {:3.1f} %'.format(i, test_accuracy * 100.0))
plt.tight_layout()
You've just been hired at a wine company and they would like you to help them build a model that predicts the quality of their wine based on several measurements. They give you a dataset with wine
validation_split=0.2. Can you converge to 100% validation accuracy?
In [69]:
ex1 = pd.read_csv('./data/wines.csv')
In [70]:
ex1.head()
Out[70]:
In [71]:
ex1.describe()
Out[71]:
In [72]:
ex1.info()
In [73]:
target = ex1['Class']
In [74]:
target.value_counts()
Out[74]:
In [75]:
features = ex1.ix[:, ex1.columns != 'Class']
In [76]:
features.info()
In [77]:
import seaborn as sns
In [78]:
#sns.pairplot(ex1, hue = 'Class')
In [79]:
from sklearn.preprocessing import MinMaxScaler
In [80]:
minmax = MinMaxScaler()
In [81]:
features = minmax.fit_transform(features)
In [82]:
pd.DataFrame(features,
columns = ['Alcohol','Malic_acid','Ash',
'Alcalinity_of_ash', 'Magnesium', 'Total_phenols',
'Flavanoids', 'Nonflavanoid_phenols', 'Proanthocyanins',
'Color_intensity','Hue','OD280-OD315_of_diluted_wines','Proline']).head()
Out[82]:
In [83]:
from keras.utils import to_categorical
In [84]:
target = pd.get_dummies(ex1['Class']).values
In [85]:
target
Out[85]:
In [86]:
ex1['Class'].value_counts()
Out[86]:
In [87]:
pd.DataFrame(target).head()
Out[87]:
In [88]:
dflist = []
optimizers = ['SGD(lr = 0.01)',
'SGD(lr = 0.01, momentum = 0.3)',
'SGD(lr = 0.01, momentum = 0.3, nesterov = True)',
'Adam(lr = 0.01)',
'Adam(lr = 0.005)',
'Adagrad(lr = 0.01)',
'RMSprop(lr = 0.01)']
from tqdm import tqdm
for opt_name in tqdm(optimizers):
K.clear_session()
model = Sequential()
model.add(Dense(130,
input_shape = (13,),
activation = 'tanh'))
model.add(Dense(3,
activation = 'softmax'))
model.compile(optimizer = eval(opt_name),
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
h = model.fit(features, target,
batch_size = 16,
epochs = 10,
verbose = 0)
dflist.append(pd.DataFrame(h.history,
index = h.epoch))
In [89]:
historydf = pd.concat(dflist,
axis = 1)
metrics_reported = dflist[0].columns
idx = pd.MultiIndex.from_product([optimizers, metrics_reported],
names = ['optimizers', 'metric'])
historydf.columns = idx
fig, ax = plt.subplots(figsize = (15, 10))
ax = plt.subplot(211)
historydf.xs('loss',
axis = 1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Loss")
ax = plt.subplot(212)
historydf.xs('acc',
axis = 1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Accuracy")
plt.xlabel("Epochs")
plt.tight_layout()
Adam optimizer with learning rate = 0.01 was chosen.
In [90]:
dflist = []
batch_sizes = [2, 4, 6, 8, 10, 12, 24]
from tqdm import tqdm
for batch_size in tqdm(batch_sizes):
K.clear_session()
model = Sequential()
model.add(Dense(130,
input_shape = (13,),
activation = 'tanh'))
model.add(Dense(3,
activation = 'softmax'))
model.compile(loss = 'categorical_crossentropy',
optimizer = Adam(lr = 0.01),
metrics = ['accuracy'])
h = model.fit(features, target,
batch_size = batch_size,
epochs = 10,
verbose = 0)
dflist.append(pd.DataFrame(h.history,
index = h.epoch))
In [91]:
historydf = pd.concat(dflist,
axis = 1)
In [92]:
historydf.head()
Out[92]:
In [93]:
metrics_reported = dflist[0].columns
metrics_reported
Out[93]:
In [94]:
idx = pd.MultiIndex.from_product([batch_sizes, metrics_reported],
names = ['optimizers', 'metric'])
historydf.columns = idx
fig, ax = plt.subplots(figsize = (15, 10))
ax = plt.subplot(211)
historydf.xs('loss',
axis = 1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Loss")
ax = plt.subplot(212)
historydf.xs('acc',
axis = 1,
level = 'metric').plot(ylim = (0,1),
ax = ax)
plt.title("Accuracy")
plt.xlabel("Epochs")
plt.tight_layout()
Batch size of 6 was chosen!
In [95]:
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(features, target, test_size = 0.2, random_state = 7)
In [96]:
X_train.shape
Out[96]:
In [97]:
Y_train.shape
Out[97]:
In [98]:
K.clear_session()
In [99]:
model = Sequential()
model.add(Dense(130,
input_shape = (13, ),
activation = 'tanh'))
model.add(Dense(3,
activation = 'softmax'))
model.compile(optimizer = Adam(lr = 0.01),
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
In [100]:
model.summary()
In [101]:
from keras.callbacks import History
history = History()
In [102]:
model.fit(X_train,
Y_train,
validation_split = 0.1,
epochs = 200,
verbose = 1, callbacks = [history])
Out[102]:
In [103]:
history.history.keys()
Out[103]:
In [104]:
# Last few values from the history
history.history['val_acc'][-5:]
Out[104]:
In [105]:
model.evaluate(X_test, Y_test)
Out[105]:
Since this dataset has 13 features we can only visualize pairs of features like we did in the Paired plot. We could however exploit the fact that a neural network is a function to extract 2 high level features to represent our data.
In [106]:
K.clear_session()
model = Sequential()
model.add(Dense(units = 8,
input_shape = (13, ),
activation = 'tanh',
name = 'Layer_1'))
model.add(Dense(units = 5,
activation = 'tanh',
name = 'Layer_2'))
model.add(Dense(units = 2,
activation = 'relu',
name = 'Layer_3'))
model.add(Dense(units = 3,
activation = 'softmax',
name = 'Output'))
model.compile(optimizer = Adam(lr = 0.01),
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
In [107]:
model.summary()
In [108]:
model.fit(X_train,
Y_train,
validation_split = 0.1,
epochs = 20,
verbose = 2)
Out[108]:
In [109]:
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
SVG(model_to_dot(model).create(prog='dot', format='svg'))
Out[109]:
In [110]:
from keras.utils import plot_model
plot_model(model, to_file='5_model_1.png', show_shapes = True)
In [111]:
from IPython.display import Image
Image('5_model_1.png')
Out[111]:
In [112]:
# Intput of the 1st layer
inp = model.layers[0].input
# Output of the 3rd layer
output = model.layers[2].output
In [113]:
feature_func = K.function([inp], [output])
In [114]:
my_features = feature_func([X_train])[0]
In [115]:
plt.scatter(my_features[:,0], my_features[:,1], c = Y_train)
Out[115]:
In [116]:
model.evaluate(X_test, Y_test)
Out[116]:
Keras functional API. So far we've always used the Sequential model API in Keras. However, Keras also offers a Functional API, which is much more powerful. You can find its documentation here. Let's see how we can leverage it.
inputssecond_to_last layer with 2 nodes
In [117]:
K.clear_session()
In [118]:
from keras.models import Model
from keras.layers import Input
In [119]:
inputs = Input(shape = (13, ))
In [120]:
x = Dense(units = 8,
kernel_initializer= 'he_normal',
activation = 'tanh')(inputs)
x = Dense(units = 5,
kernel_initializer= 'he_normal',
activation = 'tanh')(x)
second_to_last = Dense(units = 2,
kernel_initializer = 'he_normal',
activation = 'tanh')(x)
prediction = Dense(units = 3,
activation = 'softmax')(second_to_last)
In [121]:
model = Model(inputs = inputs, outputs = prediction)
model.compile(optimizer = Adam(lr = 0.005),
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
In [122]:
model.summary()
In [123]:
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
SVG(model_to_dot(model).create(prog = 'dot',
format = 'svg'))
Out[123]:
In [124]:
model.fit(X_train,
Y_train,
batch_size = 6,
validation_split = 0.1,
epochs = 20,
verbose = 2)
Out[124]:
In [125]:
features_function = K.function([inputs], [second_to_last])
In [126]:
features = features_function([X_train])[0]
In [127]:
plt.scatter(features[:, 0], features[:, 1], c = Y_train)
Out[127]:
Keras offers the possibility to call a function at each epoch. These are Callbacks, and their documentation is here. Callbacks allow us to add some neat functionality. In this exercise we'll explore a few of them.
validation_data = (X_test, y_test)EarlyStopping callback to stop your training if the val_loss doesn't improveModelCheckpoint callback to save the trained model to disk once training is finishedTensorBoard callback to output your training information to a /tmp/ subdirectory
In [128]:
X = ex1.ix[:, ex1.columns != 'Class']
X.shape
Out[128]:
In [129]:
Y = ex1['Class']
Y.shape
Out[129]:
In [130]:
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
In [131]:
X.head()
Out[131]:
In [132]:
minmax = MinMaxScaler()
X = minmax.fit_transform(X)
In [133]:
Y.head()
Out[133]:
In [134]:
Y = pd.get_dummies(Y)
In [135]:
Y.head()
Out[135]:
In [136]:
Y = Y.values
In [137]:
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size = 0.3,
random_state = 42)
In [138]:
X_train.shape, X_test.shape, Y_train.shape, Y_test.shape
Out[138]:
In [139]:
K.clear_session()
In [140]:
from keras.models import Model
from keras.layers import Input, Dense
In [141]:
inputs = Input(shape = (13, ))
In [142]:
net = Dense(units = 10,
activation = 'tanh',
kernel_initializer = 'he_normal')(inputs)
net = Dense(units = 10,
activation = 'tanh',
kernel_initializer = 'he_normal')(net)
prediction = Dense(units = 3,
activation = 'softmax')(net)
In [143]:
model = Model(inputs = inputs, outputs = prediction)
In [144]:
model.compile(optimizer = Adam(lr = 0.01),
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
In [145]:
model.summary()
In [146]:
from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard
early_stop = EarlyStopping(monitor = 'val_loss', patience = 5)
model_check = ModelCheckpoint(filepath = './tmp/Notebook_5.hdf5',
monitor = 'val_loss',
save_best_only = True)
tensorboard = TensorBoard(log_dir = './logs')
In [147]:
model.fit(X_train,
Y_train,
batch_size = 6,
epochs = 200,
validation_data = (X_test, Y_test),
verbose = 1,
callbacks = [early_stop, model_check, tensorboard])
Out[147]: