In [1]:

    

from __future__ import print_function
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.callbacks import ModelCheckpoint, EarlyStopping, LambdaCallback
from keras.optimizers import Adam, RMSprop, SGD
from keras.regularizers import l2, activity_l2

import numpy as np
import matplotlib.pyplot as plt

# reproducibility
np.random.seed(31337)

NAME="ResNet50-300x300-MaxPooling"


    

    




Using TensorFlow backend.

In [2]:

    

import h5py
import numpy as np
with h5py.File(NAME+"_codes-train.h5") as hf:
    X_train = hf["X_train"][:]
    Y_train = hf["Y_train"][:]
    X_valid = hf["X_valid"][:]
    Y_valid = hf["Y_valid"][:]
    

with h5py.File(NAME+"_codes-test.h5") as hf:
      X_test = hf["X_test"][:]


    

In [3]:

    

model = Sequential()
#model.add(Dropout(0.3, input_shape=(2048,)))
model.add(Dense(256, activation='relu', b_regularizer=l2(0.01), input_dim=2048))
model.add(Dropout(0.25))
model.add(Dense(2, activation="softmax"))

# Callbacks
early_stop_cb = EarlyStopping(monitor='val_loss', patience=50, verbose=1)

checkpoit_cb = ModelCheckpoint(NAME+"-head-Droput0.5-Dense256-Dropout.5-Dense2.h5", save_best_only=True)

# Print the batch number at the beginning of every batch.
batch_print_cb = LambdaCallback(on_batch_begin=lambda batch, logs: print(".",end=''), 
                                on_epoch_end=lambda batch, logs: print(batch))

# Plot the loss after every epoch.
plot_loss_cb = LambdaCallback(on_epoch_end=lambda epoch, logs: 
                              print (logs))
                              #plt.plot(np.arange(epoch), logs['loss']))


model.compile(loss='categorical_crossentropy',
              optimizer=SGD(lr=0.05, momentum=0.9, decay=0.005), #Adam(lr=0.008),
              metrics=['accuracy'],
             )

model.summary()


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
dense_1 (Dense)                  (None, 256)           524544      dense_input_1[0][0]              
____________________________________________________________________________________________________
dropout_1 (Dropout)              (None, 256)           0           dense_1[0][0]                    
____________________________________________________________________________________________________
dense_2 (Dense)                  (None, 2)             514         dropout_1[0][0]                  
====================================================================================================
Total params: 525,058
Trainable params: 525,058
Non-trainable params: 0
____________________________________________________________________________________________________

In [4]:

    

history = model.fit(
        np.concatenate((X_train,X_valid)),
        np.concatenate((Y_train,Y_valid)),
        nb_epoch=300,
        batch_size=4096,
        #validation_data=(X_valid,Y_valid),
        validation_split=0.33,
        callbacks=[early_stop_cb, checkpoit_cb, batch_print_cb, plot_loss_cb],
        verbose=0
        )


    

    




.....0
{'acc': 0.85743344386394893, 'loss': 0.33141314794484672, 'val_acc': 0.98591207236029843, 'val_loss': 0.04072015531829818}
.....1
{'acc': 0.98091534474207687, 'loss': 0.057068890322700717, 'val_acc': 0.9765606024093294, 'val_loss': 0.077872810605257387}
.....2
{'acc': 0.98629973162942897, 'loss': 0.048214235894477833, 'val_acc': 0.98773378691164826, 'val_loss': 0.040099762360216319}
.....3
{'acc': 0.9868381685530252, 'loss': 0.046314837605166748, 'val_acc': 0.98324022349264251, 'val_loss': 0.058949206410781663}
.....4
{'acc': 0.98851331224564198, 'loss': 0.039181413564796472, 'val_acc': 0.98518338613438328, 'val_loss': 0.05179542062031809}
.....5
{'acc': 0.98953036119436522, 'loss': 0.033958721785930651, 'val_acc': 0.98627641503892538, 'val_loss': 0.044050758831675321}
.....6
{'acc': 0.99102602377288751, 'loss': 0.02981961316604868, 'val_acc': 0.98530483379044342, 'val_loss': 0.0468248653851987}
.....7
{'acc': 0.99012862513816324, 'loss': 0.028414428781917484, 'val_acc': 0.98494049082226276, 'val_loss': 0.045166606368994229}
.....8
{'acc': 0.99078671669054275, 'loss': 0.027150865729520447, 'val_acc': 0.97947534629955202, 'val_loss': 0.066270166251130663}
.....9
{'acc': 0.9925815117847695, 'loss': 0.023574827591499418, 'val_acc': 0.98906971127308774, 'val_loss': 0.030197388875504737}
.....10
{'acc': 0.99240203281023742, 'loss': 0.023734592169639131, 'val_acc': 0.9856691767586242, 'val_loss': 0.038930429578667158}
.....11
{'acc': 0.99246186038315953, 'loss': 0.021542831832799876, 'val_acc': 0.98275443301317844, 'val_loss': 0.048465684208297777}
.....12
{'acc': 0.99323960438910053, 'loss': 0.020257732166618671, 'val_acc': 0.9887053681601301, 'val_loss': 0.029820189069986403}
.....13
{'acc': 0.99276099000332274, 'loss': 0.018415507937539419, 'val_acc': 0.98554772910256394, 'val_loss': 0.040862046737736919}
.....14
{'acc': 0.99329943142713206, 'loss': 0.019030835558171404, 'val_acc': 0.97862521270713032, 'val_loss': 0.055410009772417931}
.....15
{'acc': 0.993538735079045, 'loss': 0.018896374086571639, 'val_acc': 0.98906971112831088, 'val_loss': 0.02874931063478596}
.....16
{'acc': 0.9934190833636326, 'loss': 0.016752735702385009, 'val_acc': 0.98530483379044342, 'val_loss': 0.039830253172447767}
.....17
{'acc': 0.99407717436329179, 'loss': 0.016422773687703288, 'val_acc': 0.98457614785408198, 'val_loss': 0.040854486593610601}
.....18
{'acc': 0.99437630923964682, 'loss': 0.01589711577153196, 'val_acc': 0.98797668222376867, 'val_loss': 0.033569479731714841}
.....19
{'acc': 0.99461561471731974, 'loss': 0.014985862021723658, 'val_acc': 0.98749089159952774, 'val_loss': 0.035485058657341799}
.....20
{'acc': 0.9950343989484367, 'loss': 0.014809313874871137, 'val_acc': 0.98615496738286512, 'val_loss': 0.038709318048605966}
.....21
{'acc': 0.9950343989484367, 'loss': 0.014391282968269898, 'val_acc': 0.98736944394346748, 'val_loss': 0.036573948202447054}
.....22
{'acc': 0.99557283930246465, 'loss': 0.014093797477956771, 'val_acc': 0.98931260644043129, 'val_loss': 0.028347238985143415}
.....23
{'acc': 0.99461561289155975, 'loss': 0.014569141641716079, 'val_acc': 0.98773378691164826, 'val_loss': 0.03553235364902764}
.....24
{'acc': 0.99527370496100021, 'loss': 0.014570174249674153, 'val_acc': 0.9867622056631663, 'val_loss': 0.037778979158746055}
.....25
{'acc': 0.99581214478013758, 'loss': 0.01313898748164646, 'val_acc': 0.9884624728480097, 'val_loss': 0.032741882198349725}
.....26
{'acc': 0.99521387974872866, 'loss': 0.01299037119630538, 'val_acc': 0.987612339255588, 'val_loss': 0.035292461995856117}
.....27
{'acc': 0.99569249177385577, 'loss': 0.013522938231907221, 'val_acc': 0.98554772910256394, 'val_loss': 0.042891993687979246}
.....28
{'acc': 0.99557283747670466, 'loss': 0.012945082865102701, 'val_acc': 0.98639786269498564, 'val_loss': 0.039418040784942258}
.....29
{'acc': 0.99623092847636385, 'loss': 0.012395249428453201, 'val_acc': 0.98749089159952774, 'val_loss': 0.037011314108370573}
.....30
{'acc': 0.99527370496100021, 'loss': 0.012656097556373832, 'val_acc': 0.98834102519194944, 'val_loss': 0.034296597712663066}
.....31
{'acc': 0.9960514489669412, 'loss': 0.012250892447395183, 'val_acc': 0.98749089159952774, 'val_loss': 0.036442425774303143}
.....32
{'acc': 0.99641040798578651, 'loss': 0.011206436296075253, 'val_acc': 0.98579062441468446, 'val_loss': 0.040870419369758493}
.....33
{'acc': 0.99700867355208611, 'loss': 0.011497873742598065, 'val_acc': 0.9867622056631663, 'val_loss': 0.038754493560943321}
.....34
{'acc': 0.99647023448892746, 'loss': 0.011435107426534566, 'val_acc': 0.9884624728480097, 'val_loss': 0.033309980488558953}
.....35
{'acc': 0.99676936646974124, 'loss': 0.011095834043653665, 'val_acc': 0.98749089159952774, 'val_loss': 0.035904026656568354}
.....36
{'acc': 0.99718815252661808, 'loss': 0.010975851620191716, 'val_acc': 0.98724799628740723, 'val_loss': 0.036923276227467959}
.....37
{'acc': 0.99670954103638165, 'loss': 0.010260616561145509, 'val_acc': 0.98906971127308774, 'val_loss': 0.03212769052843055}
.....38
{'acc': 0.99670953996660039, 'loss': 0.011068331321710879, 'val_acc': 0.98906971127308774, 'val_loss': 0.03082204005484851}
.....39
{'acc': 0.99700867301719542, 'loss': 0.010396443949435646, 'val_acc': 0.98664075800710604, 'val_loss': 0.039543945650858066}
.....40
{'acc': 0.99658988749520916, 'loss': 0.01028901029485711, 'val_acc': 0.9856691767586242, 'val_loss': 0.043139644564356303}
.....41
{'acc': 0.99706850134609637, 'loss': 0.010196580165858411, 'val_acc': 0.98712654863134708, 'val_loss': 0.0370147713659775}
.....42
{'acc': 0.99718815252661808, 'loss': 0.0096191068708667074, 'val_acc': 0.9884624728480097, 'val_loss': 0.03308556301445529}
.....43
{'acc': 0.99748728740297321, 'loss': 0.010159281014475661, 'val_acc': 0.98809812987982892, 'val_loss': 0.035315131861331943}
.....44
{'acc': 0.99676936936528249, 'loss': 0.0096671058905274439, 'val_acc': 0.98627641503892538, 'val_loss': 0.040892622753243034}
.....45
{'acc': 0.99730780789355056, 'loss': 0.00975818633760186, 'val_acc': 0.98700510097528682, 'val_loss': 0.038194766666722987}
.....46
{'acc': 0.99688902001091362, 'loss': 0.0094230802867905916, 'val_acc': 0.98809812987982892, 'val_loss': 0.034161773531767588}
.....47
{'acc': 0.99706849898544569, 'loss': 0.0090591748340616424, 'val_acc': 0.987612339255588, 'val_loss': 0.03579339668252441}
.....48
{'acc': 0.99748728686808252, 'loss': 0.0093127809215037272, 'val_acc': 0.98736944394346748, 'val_loss': 0.036538854617791064}
.....49
{'acc': 0.9972479813904096, 'loss': 0.0087241850567704692, 'val_acc': 0.98712654863134708, 'val_loss': 0.037627772086738585}
.....50
{'acc': 0.99754711154546349, 'loss': 0.0087774533465295144, 'val_acc': 0.98724799628740723, 'val_loss': 0.037432295659125821}
.....51
{'acc': 0.99766676637750529, 'loss': 0.0081731067684980788, 'val_acc': 0.9865193103510459, 'val_loss': 0.03959046706886641}
.....52
{'acc': 0.99796589653255907, 'loss': 0.0078742972321890793, 'val_acc': 0.98724799628740723, 'val_loss': 0.03836615652729071}
.....53
{'acc': 0.99796589706744976, 'loss': 0.0087282603634997753, 'val_acc': 0.98809812987982892, 'val_loss': 0.03584886343191919}
.....54
{'acc': 0.99754711337122348, 'loss': 0.0081092725764077486, 'val_acc': 0.98882681581619036, 'val_loss': 0.033842485364574466}
.....55
{'acc': 0.99784624588692794, 'loss': 0.0086541026190893895, 'val_acc': 0.98858392050406985, 'val_loss': 0.034562440881791091}
.....56
{'acc': 0.99772659288064613, 'loss': 0.0079019861259324135, 'val_acc': 0.98724799628740723, 'val_loss': 0.037857178462009158}
.....57
{'acc': 0.99808555007373145, 'loss': 0.0078455980928437002, 'val_acc': 0.98712654863134708, 'val_loss': 0.038824555677408612}
.....58
{'acc': 0.99814537657687241, 'loss': 0.0075810102748119328, 'val_acc': 0.98773378691164826, 'val_loss': 0.035993803821853389}
.....59
{'acc': 0.99766676401685461, 'loss': 0.0081841077196578658, 'val_acc': 0.98749089159952774, 'val_loss': 0.037944970560618213}
.....60
{'acc': 0.99790607002941822, 'loss': 0.0079690455473962786, 'val_acc': 0.9865193103510459, 'val_loss': 0.040948053911049623}
.....61
{'acc': 0.99778641755802699, 'loss': 0.008063452220136021, 'val_acc': 0.9865193103510459, 'val_loss': 0.040574135453790204}
.....62
{'acc': 0.99832485555140449, 'loss': 0.0074407918790882974, 'val_acc': 0.98664075800710604, 'val_loss': 0.039969758142146899}
.....63
{'acc': 0.99796589889320975, 'loss': 0.0077332890648233238, 'val_acc': 0.98749089159952774, 'val_loss': 0.037354926146405801}
.....64
{'acc': 0.99832485791205505, 'loss': 0.0069875726276563471, 'val_acc': 0.98785523456770841, 'val_loss': 0.035493907630660737}
.....65
{'acc': 0.99832485555140449, 'loss': 0.0067623249519780174, 'val_acc': 0.98773378691164826, 'val_loss': 0.036739028107129694}
.....66
{'acc': 0.99844450855768618, 'loss': 0.0072325443986869043, 'val_acc': 0.98736944394346748, 'val_loss': 0.038159927654127374}
.....67
{'acc': 0.99796589653255907, 'loss': 0.007387764960852812, 'val_acc': 0.98736944394346748, 'val_loss': 0.037602956414237193}
.....68
{'acc': 0.99844450855768618, 'loss': 0.0066304514564867953, 'val_acc': 0.98749089159952774, 'val_loss': 0.036692997685459187}
.....69
{'acc': 0.99850433742147771, 'loss': 0.0066984670206619136, 'val_acc': 0.98736944394346748, 'val_loss': 0.037428544243837923}
.....70
{'acc': 0.99808555007373145, 'loss': 0.0073292984137706615, 'val_acc': 0.98736944394346748, 'val_loss': 0.037388625050057245}
.....71
{'acc': 0.99766676508663588, 'loss': 0.0070466867935290211, 'val_acc': 0.98809812987982892, 'val_loss': 0.035698255920818427}
.....72
{'acc': 0.99856416156396799, 'loss': 0.0068148079833459871, 'val_acc': 0.98700510097528682, 'val_loss': 0.040098725723313973}
.....73
{'acc': 0.99850433506082714, 'loss': 0.006496427364027807, 'val_acc': 0.98700510097528682, 'val_loss': 0.040568068552964055}
Epoch 00073: early stopping

In [ ]:

    




    

In [5]:

    

score=model.evaluate(X_valid,Y_valid, verbose=0)
print("OOS %s: %.2f%%" % (model.metrics_names[1], score[1]*100))
print("OOS %s: %.2f" % (model.metrics_names[0], score[0]))

#print("min(val los)",np.min(history.history['val_loss']))


    

    




OOS acc: 99.00%
OOS loss: 0.03

In [6]:

    

# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='lower right')
plt.show()


# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss (log scale)')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper right')
plt.yscale('log')
plt.show()


    

In [7]:

    

from keras.models import load_model
from sklearn.metrics import log_loss

model = load_model(NAME+"-head-Droput0.5-Dense256-Dropout.5-Dense2.h5")


    

In [8]:

    

Y_train_preds=model.predict(X_train)
print(log_loss(Y_train,Y_train_preds))

Y_valid_preds=model.predict(X_valid)
print(log_loss(Y_valid,Y_valid_preds))


    

    




0.0176539292443
0.0231948970593

In [9]:

    

for i in range(1,10):
    m=i/100.0
    print(m,log_loss(Y_valid,Y_valid_preds.clip(min=m, max=1-m)))
    
for i in range(1,10):
    m=i/100.0
    print(m,log_loss(Y_train,Y_train_preds.clip(min=m, max=1-m)))


    

    




0.01 0.0322891618269
0.02 0.041481228807
0.03 0.0507627845732
0.04 0.0603102860004
0.05 0.0701100940984
0.06 0.0801007732488
0.07 0.090243214041
0.08 0.100510325398
0.09 0.110956470706
0.01 0.0263734755352
0.02 0.0357606290104
0.03 0.0453413155537
0.04 0.0550939046605
0.05 0.0650187074275
0.06 0.0750939386557
0.07 0.0853261290293
0.08 0.0956843544155
0.09 0.106167971919

In [10]:

    

import re
datagen = ImageDataGenerator()
test_batches = datagen.flow_from_directory("test", model.input_shape[1:3], shuffle=False, batch_size=32, class_mode=None)

ids = [re.split('/|\.',x)[1] for x in test_batches.filenames]


    

    




Found 12500 images belonging to 1 classes.

In [11]:

    

Y_test = model.predict(X_test)


    

In [12]:

    

Y_test_csv=np.column_stack((ids,Y_test[:,1]))
np.savetxt('Y_test_'+NAME+'-head-Droput0.5-Dense256-Dropout.5-Dense2.csv',Y_test_csv, fmt="%s,%s", header=("id,label"),comments='')


    

In [13]:

    

Y_test_csv=np.column_stack((ids,Y_test[:,1].clip(min=0.02, max=0.98)))
np.savetxt('Y_test_'+NAME+'-head-Droput0.5-Dense256-Dropout.5-Dense2-clipped.csv',Y_test_csv, fmt="%s,%s", header=("id,label"),comments='')


    

In [ ]: