notebook.community

Edit and run



In [1]:

    
from utils import *









    



Using TensorFlow backend.



In [2]:

    
path = 'data/mnist/'
# path = 'data/mnist/sample' # sample path



In [3]:

    
train = pd.read_csv(path + 'train.csv')
test = pd.read_csv(path + 'test.csv')



In [4]:

    
train = train.values
test = test.values
# 打乱
train = np.random.permutation(train)



In [5]:

    
valid = train[:2000, :]
valid_data = valid[:, 1:]
valid_label = onehot(valid[:, 0])



In [6]:

    
train = train[2000:, :]



In [7]:

    
train_data = train[:, 1:]
train_label = onehot(train[:, 0])



In [8]:

    
train_data.shape, train_label.shape, valid_label.shape, valid_data.shape









    Out[8]:





((40000, 784), (40000, 10), (2000, 10), (2000, 784))



In [9]:

    
def reshape(data, target_size): return np.reshape(data, target_size)



In [10]:

    
train_data = reshape(train_data, [40000, 1, 28, 28])
valid_data = reshape(valid_data, [2000, 1, 28, 28])
train_data.shape, train_label.shape, valid_label.shape, valid_data.shape









    Out[10]:





((40000, 1, 28, 28), (40000, 10), (2000, 10), (2000, 1, 28, 28))



In [11]:

    
mean_px = train_data.mean().astype(np.float32)
std_px = train_data.std().astype(np.float32)



In [12]:

    
def norm_input(x): return (x-mean_px)/std_px



In [13]:

    
BATCH_SIZE = 64

线性模型



In [14]:

    
def get_lin_model():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Flatten(),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



In [15]:

    
lm = get_lin_model()



In [16]:

    
gen = image.ImageDataGenerator()
batches = gen.flow(train_data, train_label, batch_size=BATCH_SIZE)
val_batches = gen.flow(valid_data, valid_label, batch_size=BATCH_SIZE)



In [17]:

    
lm.fit_generator(
    batches, 
    samples_per_epoch=len(train_data) / 20, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=len(valid_data) / 20
)









    



Epoch 1/1
2000/2000 [==============================] - 1s - loss: 1.4026 - acc: 0.5315 - val_loss: 0.7518 - val_acc: 0.7667






    Out[17]:





<keras.callbacks.History at 0x7fece25f22e8>



In [18]:

    
lm.fit_generator(
    batches, 
    samples_per_epoch=len(train_data) / 20, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=len(valid_data) / 20
)









    



Epoch 1/1
2000/2000 [==============================] - 0s - loss: 0.6715 - acc: 0.7840 - val_loss: 0.5619 - val_acc: 0.8417






    Out[18]:





<keras.callbacks.History at 0x7fece25f2400>

多层神经网络



In [19]:

    
def get_fc_model():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Flatten(),
        Dense(512, activation='relu'),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(lr=.01), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



In [20]:

    
fc = get_fc_model()



In [21]:

    
fc.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 6s - loss: 0.8576 - acc: 0.8798 - val_loss: 0.1976 - val_acc: 0.9455






    Out[21]:





<keras.callbacks.History at 0x7fc16fae3080>



In [22]:

    
def get_model():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Convolution2D(32,3,3, activation='relu'),
        Convolution2D(32,3,3, activation='relu'),
        MaxPooling2D(),
        Convolution2D(64,3,3, activation='relu'),
        Convolution2D(64,3,3, activation='relu'),
        MaxPooling2D(),
        Flatten(),
        Dense(512, activation='relu'),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



In [23]:

    
model = get_model()



In [24]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 15s - loss: 0.1375 - acc: 0.9565 - val_loss: 0.0494 - val_acc: 0.9855






    Out[24]:





<keras.callbacks.History at 0x7fc16e44db38>

数据增强



In [25]:

    
model = get_model()



In [17]:

    
gen = image.ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,
                               height_shift_range=0.08, zoom_range=0.08)
batches = gen.flow(train_data, train_label, batch_size=BATCH_SIZE)
val_batches = gen.flow(valid_data, valid_label, batch_size=BATCH_SIZE)



In [27]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 15s - loss: 0.2505 - acc: 0.9200 - val_loss: 0.0979 - val_acc: 0.9680






    Out[27]:





<keras.callbacks.History at 0x7fc1444c2fd0>



In [18]:

    
def get_model_bn_do():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Convolution2D(32,3,3, activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(32,3,3, activation='relu'),
        MaxPooling2D(),
        BatchNormalization(axis=1),
        Convolution2D(64,3,3, activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(64,3,3, activation='relu'),
        MaxPooling2D(),
        Flatten(),
#         BatchNormalization(),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dropout(0.5),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



In [19]:

    
model = get_model_bn_do()



In [20]:

    
model.optimizer.lr=0.01



In [21]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=4, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/4
40000/40000 [==============================] - 44s - loss: 0.2628 - acc: 0.9213 - val_loss: 0.1941 - val_acc: 0.9415
Epoch 2/4
40000/40000 [==============================] - 42s - loss: 0.1200 - acc: 0.9631 - val_loss: 0.1249 - val_acc: 0.9605
Epoch 3/4
40000/40000 [==============================] - 43s - loss: 0.0963 - acc: 0.9710 - val_loss: 0.0853 - val_acc: 0.9725
Epoch 4/4
40000/40000 [==============================] - 44s - loss: 0.0889 - acc: 0.9731 - val_loss: 0.1096 - val_acc: 0.9655






    Out[21]:





<keras.callbacks.History at 0x7efc5a028d30>



In [22]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 44s - loss: 0.0858 - acc: 0.9743 - val_loss: 0.1073 - val_acc: 0.9695






    Out[22]:





<keras.callbacks.History at 0x7efcc758ae10>



In [23]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 42s - loss: 0.0774 - acc: 0.9769 - val_loss: 0.0812 - val_acc: 0.9790






    Out[23]:





<keras.callbacks.History at 0x7efc5a006d68>



In [24]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 41s - loss: 0.0781 - acc: 0.9770 - val_loss: 0.1137 - val_acc: 0.9680






    Out[24]:





<keras.callbacks.History at 0x7efc5a006d30>



In [25]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 43s - loss: 0.0763 - acc: 0.9774 - val_loss: 0.0741 - val_acc: 0.9805






    Out[25]:





<keras.callbacks.History at 0x7efc59ff4d30>



In [26]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=2, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/2
40000/40000 [==============================] - 43s - loss: 0.0737 - acc: 0.9782 - val_loss: 0.1035 - val_acc: 0.9730
Epoch 2/2
40000/40000 [==============================] - 44s - loss: 0.0716 - acc: 0.9787 - val_loss: 0.0597 - val_acc: 0.9825






    Out[26]:





<keras.callbacks.History at 0x7efc5ad49160>



In [27]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=2, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/2
40000/40000 [==============================] - 44s - loss: 0.0690 - acc: 0.9806 - val_loss: 0.0884 - val_acc: 0.9750
Epoch 2/2
40000/40000 [==============================] - 43s - loss: 0.0673 - acc: 0.9810 - val_loss: 0.0829 - val_acc: 0.9760.9






    Out[27]:





<keras.callbacks.History at 0x7efc59fa67b8>



In [28]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=2, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/2
40000/40000 [==============================] - 43s - loss: 0.0606 - acc: 0.9827 - val_loss: 0.0936 - val_acc: 0.9825
Epoch 2/2
40000/40000 [==============================] - 44s - loss: 0.0641 - acc: 0.9819 - val_loss: 0.0726 - val_acc: 0.9805






    Out[28]:





<keras.callbacks.History at 0x7efc59fa6518>



In [29]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 44s - loss: 0.0612 - acc: 0.9832 - val_loss: 0.0886 - val_acc: 0.9800






    Out[29]:





<keras.callbacks.History at 0x7efc59fa6780>



In [30]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=3, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/3
40000/40000 [==============================] - 42s - loss: 0.0605 - acc: 0.9828 - val_loss: 0.0557 - val_acc: 0.9840
Epoch 2/3
40000/40000 [==============================] - 42s - loss: 0.0586 - acc: 0.9833 - val_loss: 0.0718 - val_acc: 0.9805
Epoch 3/3
40000/40000 [==============================] - 41s - loss: 0.0510 - acc: 0.9866 - val_loss: 0.0730 - val_acc: 0.9815






    Out[30]:





<keras.callbacks.History at 0x7efc59fa6748>



In [31]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=3, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/3
40000/40000 [==============================] - 42s - loss: 0.0654 - acc: 0.9829 - val_loss: 0.1056 - val_acc: 0.9770
Epoch 2/3
40000/40000 [==============================] - 43s - loss: 0.0544 - acc: 0.9853 - val_loss: 0.0573 - val_acc: 0.9845
Epoch 3/3
40000/40000 [==============================] - 44s - loss: 0.0552 - acc: 0.9850 - val_loss: 0.0862 - val_acc: 0.9785






    Out[31]:





<keras.callbacks.History at 0x7efc59fa6438>



In [32]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=3, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/3
40000/40000 [==============================] - 43s - loss: 0.0508 - acc: 0.9868 - val_loss: 0.0564 - val_acc: 0.9870
Epoch 2/3
40000/40000 [==============================] - 42s - loss: 0.0525 - acc: 0.9855 - val_loss: 0.0588 - val_acc: 0.9835
Epoch 3/3
40000/40000 [==============================] - 43s - loss: 0.0507 - acc: 0.9861 - val_loss: 0.0673 - val_acc: 0.9820






    Out[32]:





<keras.callbacks.History at 0x7efc59fa6940>



In [36]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=3, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/3
40000/40000 [==============================] - 42s - loss: 0.0462 - acc: 0.9883 - val_loss: 0.0997 - val_acc: 0.9815
Epoch 2/3
40000/40000 [==============================] - 44s - loss: 0.0489 - acc: 0.9883 - val_loss: 0.0764 - val_acc: 0.9830
Epoch 3/3
40000/40000 [==============================] - 42s - loss: 0.0490 - acc: 0.9885 - val_loss: 0.0863 - val_acc: 0.9820






    Out[36]:





<keras.callbacks.History at 0x7efc59fa6b00>



In [39]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=3, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/3
40000/40000 [==============================] - 42s - loss: 0.0476 - acc: 0.9885 - val_loss: 0.0827 - val_acc: 0.9855
Epoch 2/3
40000/40000 [==============================] - 42s - loss: 0.0474 - acc: 0.9895 - val_loss: 0.0677 - val_acc: 0.9850
Epoch 3/3
40000/40000 [==============================] - 42s - loss: 0.0431 - acc: 0.9896 - val_loss: 0.0855 - val_acc: 0.9855






    Out[39]:





<keras.callbacks.History at 0x7efc59fb0208>



In [40]:

    
model.optimizer.lr=0.001



In [42]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=4, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/4
40000/40000 [==============================] - 43s - loss: 0.0437 - acc: 0.9896 - val_loss: 0.1190 - val_acc: 0.9810
Epoch 2/4
40000/40000 [==============================] - 43s - loss: 0.0461 - acc: 0.9899 - val_loss: 0.0805 - val_acc: 0.9845
Epoch 3/4
40000/40000 [==============================] - 42s - loss: 0.0431 - acc: 0.9907 - val_loss: 0.0652 - val_acc: 0.9855
Epoch 4/4
40000/40000 [==============================] - 44s - loss: 0.0450 - acc: 0.9905 - val_loss: 0.1064 - val_acc: 0.9820






    Out[42]:





<keras.callbacks.History at 0x7efc59fb00b8>



In [43]:

    
model.optimizer.lr=0.0001



In [44]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=1, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/1
40000/40000 [==============================] - 42s - loss: 0.0449 - acc: 0.9904 - val_loss: 0.0614 - val_acc: 0.9860






    Out[44]:





<keras.callbacks.History at 0x7efc59fb0940>



In [49]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    nb_epoch=2, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/2
40000/40000 [==============================] - 43s - loss: 0.0401 - acc: 0.9912 - val_loss: 0.0896 - val_acc: 0.9840
Epoch 2/2
40000/40000 [==============================] - 42s - loss: 0.0473 - acc: 0.9899 - val_loss: 0.1147 - val_acc: 0.9800






    Out[49]:





<keras.callbacks.History at 0x7efc59fb09b0>



In [67]:

    
model.optimizer.lr=0.001



In [74]:

    
model.fit_generator(
    batches, 
    samples_per_epoch=batches.n, 
    #samples_per_epoch=2000, 
    nb_epoch=10, 
    validation_data=val_batches,
    nb_val_samples=val_batches.n
)









    



Epoch 1/10
40000/40000 [==============================] - 43s - loss: 0.0457 - acc: 0.9922 - val_loss: 0.0570 - val_acc: 0.9945
Epoch 2/10
40000/40000 [==============================] - 42s - loss: 0.0451 - acc: 0.9921 - val_loss: 0.0839 - val_acc: 0.9880
Epoch 3/10
40000/40000 [==============================] - 42s - loss: 0.0528 - acc: 0.9906 - val_loss: 0.1021 - val_acc: 0.9865
Epoch 4/10
40000/40000 [==============================] - 42s - loss: 0.0455 - acc: 0.9922 - val_loss: 0.0815 - val_acc: 0.9875
Epoch 5/10
40000/40000 [==============================] - 42s - loss: 0.0445 - acc: 0.9917 - val_loss: 0.1078 - val_acc: 0.9845
Epoch 6/10
40000/40000 [==============================] - 42s - loss: 0.0447 - acc: 0.9920 - val_loss: 0.1327 - val_acc: 0.9825
Epoch 7/10
40000/40000 [==============================] - 42s - loss: 0.0399 - acc: 0.9930 - val_loss: 0.1032 - val_acc: 0.9870
Epoch 8/10
40000/40000 [==============================] - 42s - loss: 0.0450 - acc: 0.9922 - val_loss: 0.1163 - val_acc: 0.9860
Epoch 9/10
40000/40000 [==============================] - 42s - loss: 0.0465 - acc: 0.9921 - val_loss: 0.0966 - val_acc: 0.9875
Epoch 10/10
40000/40000 [==============================] - 42s - loss: 0.0506 - acc: 0.9924 - val_loss: 0.1040 - val_acc: 0.9875






    Out[74]:





<keras.callbacks.History at 0x7efc59f11da0>



In [75]:

    
preds = model.predict(reshape(test, [28000, 1, 28, 28]))



In [76]:

    
pre = np.argmax(preds, 1)



In [77]:

    
pre[:5]









    Out[77]:





array([2, 0, 9, 0, 3])



In [78]:

    
conv_y_submission = pd.DataFrame({
        'ImageId': range(1, 28001), 
        'Label': pre 
    })



In [79]:

    
conv_y_submission.to_csv('./conv_submission9.csv', index=False)



In [80]:

    
# 保存权重
path = 'models/mnist/'
model.save_weights(path + 'final1.h5')



In [81]:

    
# 载入权重
model.load_weights(path + 'final1.h5')
preds = model.predict(reshape(test, [28000, 1, 28, 28]))
pre = np.argmax(preds, 1)



In [82]:

    
pre[:5]









    Out[82]:





array([2, 0, 9, 0, 3])



In [ ]: