In [1]:

    

from __future__ import print_function
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.callbacks import ModelCheckpoint, EarlyStopping, LambdaCallback
from keras.optimizers import Adam

import numpy as np
import matplotlib.pyplot as plt

# reproducibility
np.random.seed(31337)

NAME="ResNet50-300x300-supersimple"


    

    




Using TensorFlow backend.

In [2]:

    

import h5py
import numpy as np
with h5py.File(NAME+"_codes-train.h5") as hf:
    X_train = hf["X_train"][:]
    Y_train = hf["Y_train"][:]
    X_valid = hf["X_valid"][:]
    Y_valid = hf["Y_valid"][:]
    

with h5py.File(NAME+"_codes-test.h5") as hf:
      X_test = hf["X_test"][:]


    

In [3]:

    

model = Sequential()
#model.add(Dropout(0.1, input_shape=(2048,)))
# model.add(Dense(256, activation='relu', input_dim=2048))
# model.add(Dropout(0.5))
model.add(Dense(2, input_dim=2048, activation="softmax"))

# Callbacks
early_stop_cb = EarlyStopping(monitor='val_loss', patience=20, verbose=1)

checkpoit_cb = ModelCheckpoint(NAME+"-head-Dense2.h5", save_best_only=True)

# Print the batch number at the beginning of every batch.
batch_print_cb = LambdaCallback(on_batch_begin=lambda batch, logs: print(".",end=''), 
                                on_epoch_end=lambda batch, logs: print(batch))

# Plot the loss after every epoch.
plot_loss_cb = LambdaCallback(on_epoch_end=lambda epoch, logs: 
                              print (logs))
                              #plt.plot(np.arange(epoch), logs['loss']))


model.compile(loss='categorical_crossentropy',
              optimizer=Adam(lr=0.00001),
              metrics=['accuracy'],
             )

model.summary()
X_train.shape[1:]


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
dense_1 (Dense)                  (None, 2)             4098        dense_input_1[0][0]              
====================================================================================================
Total params: 4,098
Trainable params: 4,098
Non-trainable params: 0
____________________________________________________________________________________________________






    
Out[3]:





(2048,)

In [4]:

    

history = model.fit(
        X_train,
        Y_train,
        nb_epoch=300,
        batch_size=1024,
        #validation_data=(X_valid,Y_valid),
        validation_split=0.33,
        callbacks=[early_stop_cb, checkpoit_cb, batch_print_cb, plot_loss_cb],
        verbose=0
        )


    

    




.......................0
0 {'acc': 0.63828489399043564, 'loss': 0.68735777499852468, 'val_acc': 0.76599994134902949, 'val_loss': 0.51126053714752195}
.......................1
1 {'acc': 0.83323892154819934, 'loss': 0.4131021931948623, 'val_acc': 0.88499994993209841, 'val_loss': 0.32560990691184999}
.......................2
2 {'acc': 0.90936424196756005, 'loss': 0.27649790796871421, 'val_acc': 0.93249997758865355, 'val_loss': 0.2337777774333954}
.......................3
3 {'acc': 0.94047671102730501, 'loss': 0.20576188165776127, 'val_acc': 0.94699999713897709, 'val_loss': 0.18337068402767182}
.......................4
4 {'acc': 0.956642990881303, 'loss': 0.16458701465746858, 'val_acc': 0.95749998998641972, 'val_loss': 0.15145972108840941}
.......................5
5 {'acc': 0.96391999712956122, 'loss': 0.13812038407967944, 'val_acc': 0.96350001239776606, 'val_loss': 0.13021751379966737}
.......................6
6 {'acc': 0.96988975644615505, 'loss': 0.11987400691034415, 'val_acc': 0.96999998521804809, 'val_loss': 0.11443098676204681}
.......................7
7 {'acc': 0.97324502232796883, 'loss': 0.10633987455954973, 'val_acc': 0.97299995613098145, 'val_loss': 0.10281830263137817}
.......................8
8 {'acc': 0.97524946763097931, 'loss': 0.096084662327538276, 'val_acc': 0.97500001430511474, 'val_loss': 0.093614681720733645}
.......................9
9 {'acc': 0.9777332356076982, 'loss': 0.088020766300920608, 'val_acc': 0.97450000095367428, 'val_loss': 0.086280018866062166}
.......................10
10 {'acc': 0.9786918827576172, 'loss': 0.081476647787802864, 'val_acc': 0.9774999561309814, 'val_loss': 0.080164462804794306}
.......................11
11 {'acc': 0.97982483029838241, 'loss': 0.07609757751322524, 'val_acc': 0.9794999561309814, 'val_loss': 0.075227080702781682}
.......................12
12 {'acc': 0.98047845340709061, 'loss': 0.071535680215625425, 'val_acc': 0.98049996948242191, 'val_loss': 0.070766219556331639}
.......................13
13 {'acc': 0.98161140032710947, 'loss': 0.067693109718928585, 'val_acc': 0.98099996280670165, 'val_loss': 0.06723329377174378}
.......................14
14 {'acc': 0.98226502405656413, 'loss': 0.064325088469754835, 'val_acc': 0.98149999856948855, 'val_loss': 0.063910866796970361}
.......................15
15 {'acc': 0.98318009603734657, 'loss': 0.061393084679079904, 'val_acc': 0.98099996280670165, 'val_loss': 0.061124111533164979}
.......................16
16 {'acc': 0.98387729379314082, 'loss': 0.058865589863149173, 'val_acc': 0.98099996280670165, 'val_loss': 0.058592080414295193}
.......................17
17 {'acc': 0.98435661731875224, 'loss': 0.056525651049463832, 'val_acc': 0.98099996280670165, 'val_loss': 0.056414710104465485}
.......................18
18 {'acc': 0.98483594160795973, 'loss': 0.054506199962699627, 'val_acc': 0.98149996948242191, 'val_loss': 0.054425508499145507}
.......................19
19 {'acc': 0.9853152657543176, 'loss': 0.052657258928944128, 'val_acc': 0.98249995374679566, 'val_loss': 0.052651556730270388}
.......................20
20 {'acc': 0.98553313974295464, 'loss': 0.050964470846099043, 'val_acc': 0.98249998283386231, 'val_loss': 0.051065981447696689}
.......................21
21 {'acc': 0.98596888943442418, 'loss': 0.049429854972309846, 'val_acc': 0.98349996042251586, 'val_loss': 0.049469659268856048}
.......................22
22 {'acc': 0.98636106295681003, 'loss': 0.048027090173766752, 'val_acc': 0.98399996042251592, 'val_loss': 0.048216637790203097}
.......................23
23 {'acc': 0.98666608699802072, 'loss': 0.046745192969739863, 'val_acc': 0.98449996042251586, 'val_loss': 0.046906831949949261}
.......................24
24 {'acc': 0.98684038696031828, 'loss': 0.0455606147634567, 'val_acc': 0.98449996042251586, 'val_loss': 0.045843184232711789}
.......................25
25 {'acc': 0.98692753534804467, 'loss': 0.044483633451152409, 'val_acc': 0.98449996042251586, 'val_loss': 0.044674824923276901}
.......................26
26 {'acc': 0.9875375856173636, 'loss': 0.04347089516012189, 'val_acc': 0.98499996042251592, 'val_loss': 0.043683980584144591}
.......................27
27 {'acc': 0.98762473362588932, 'loss': 0.042548678529990294, 'val_acc': 0.98549996042251586, 'val_loss': 0.042875763356685637}
.......................28
28 {'acc': 0.98753758385901491, 'loss': 0.041564859110027483, 'val_acc': 0.98499996042251592, 'val_loss': 0.041963043510913851}
.......................29
29 {'acc': 0.98788618387711158, 'loss': 0.040711626048003456, 'val_acc': 0.98549996042251586, 'val_loss': 0.04120485642552376}
.......................30
30 {'acc': 0.98792975752425038, 'loss': 0.039928716219384729, 'val_acc': 0.98549996042251586, 'val_loss': 0.04043063509464264}
.......................31
31 {'acc': 0.98827835892149496, 'loss': 0.039204011147287374, 'val_acc': 0.98549996042251586, 'val_loss': 0.03975957077741623}
.......................32
32 {'acc': 0.98849623115178342, 'loss': 0.038478435273120913, 'val_acc': 0.98599996042251592, 'val_loss': 0.039256265103816988}
.......................33
33 {'acc': 0.98853980698581978, 'loss': 0.037802677513268823, 'val_acc': 0.98599996042251592, 'val_loss': 0.038586802393198012}
.......................34
34 {'acc': 0.98875768183155455, 'loss': 0.037131394418754723, 'val_acc': 0.98599996709823612, 'val_loss': 0.037852207362651824}
.......................35
35 {'acc': 0.98888840648186604, 'loss': 0.036539362139713844, 'val_acc': 0.98649996709823606, 'val_loss': 0.037380474478006361}
.......................36
36 {'acc': 0.98901913098932803, 'loss': 0.035957668903192262, 'val_acc': 0.98749998044967646, 'val_loss': 0.036839545667171479}
.......................37
37 {'acc': 0.98932415531623796, 'loss': 0.035395570224184859, 'val_acc': 0.98699996709823612, 'val_loss': 0.036442669838666918}
.......................38
38 {'acc': 0.98923700540651394, 'loss': 0.034896783264735914, 'val_acc': 0.98799998044967652, 'val_loss': 0.03593157896399498}
.......................39
39 {'acc': 0.98932415517338834, 'loss': 0.034376750110474759, 'val_acc': 0.98799998044967652, 'val_loss': 0.035483815878629685}
.......................40
40 {'acc': 0.98932415503053872, 'loss': 0.033883654210654639, 'val_acc': 0.98799998044967652, 'val_loss': 0.035187417328357694}
.......................41
41 {'acc': 0.98949845399288905, 'loss': 0.03341996645127731, 'val_acc': 0.98799998044967652, 'val_loss': 0.034683665007352829}
.......................42
42 {'acc': 0.98967275381233699, 'loss': 0.032946659425983162, 'val_acc': 0.98799998044967652, 'val_loss': 0.034247032046318057}
.......................43
43 {'acc': 0.98980347779774869, 'loss': 0.032535673239860015, 'val_acc': 0.98799998044967652, 'val_loss': 0.03387485647201538}
.......................44
44 {'acc': 0.98993420339865934, 'loss': 0.032148675200188465, 'val_acc': 0.98799995136260987, 'val_loss': 0.033551775723695755}
.......................45
45 {'acc': 0.99028280232330723, 'loss': 0.03171062287803704, 'val_acc': 0.98949997138977053, 'val_loss': 0.03308041051030159}
.......................46
46 {'acc': 0.99010850212465851, 'loss': 0.031350308636628632, 'val_acc': 0.98899996471405027, 'val_loss': 0.032869191318750379}
.......................47
47 {'acc': 0.99023922663212049, 'loss': 0.030947952730821168, 'val_acc': 0.98950000047683717, 'val_loss': 0.032483145058155062}
.......................48
48 {'acc': 0.99050067674049314, 'loss': 0.030617413764202732, 'val_acc': 0.98949997138977053, 'val_loss': 0.032214557528495787}
.......................49
49 {'acc': 0.99036995237588088, 'loss': 0.030237713513196311, 'val_acc': 0.98949997138977053, 'val_loss': 0.031959339290857314}
.......................50
50 {'acc': 0.99032637763529319, 'loss': 0.029902333307415058, 'val_acc': 0.98949997138977053, 'val_loss': 0.031771584779024122}
.......................51
51 {'acc': 0.99071855115767915, 'loss': 0.029568156810961359, 'val_acc': 0.99000000715255743, 'val_loss': 0.031306338667869568}
.......................52
52 {'acc': 0.99084927471454187, 'loss': 0.029225149349578119, 'val_acc': 0.98949997138977053, 'val_loss': 0.031093785524368287}
.......................53
53 {'acc': 0.99102357534173946, 'loss': 0.028911609883106915, 'val_acc': 0.98999997806549067, 'val_loss': 0.030951213806867599}
.......................54
54 {'acc': 0.99080570063885431, 'loss': 0.028602425202833157, 'val_acc': 0.98999997806549067, 'val_loss': 0.03065786737203598}
.......................55
55 {'acc': 0.99137217388718657, 'loss': 0.028309425951651569, 'val_acc': 0.98949997138977053, 'val_loss': 0.030492779284715651}
.......................56
56 {'acc': 0.99119787449628738, 'loss': 0.028056795646703665, 'val_acc': 0.98999997806549067, 'val_loss': 0.030283585697412491}
.......................57
57 {'acc': 0.99124144909402545, 'loss': 0.027788599275412593, 'val_acc': 0.98999997806549067, 'val_loss': 0.030022456824779511}
.......................58
58 {'acc': 0.99141574943552369, 'loss': 0.027461098864414193, 'val_acc': 0.98999997806549067, 'val_loss': 0.029881281733512878}
.......................59
59 {'acc': 0.99159004968352049, 'loss': 0.027214481415008481, 'val_acc': 0.98999997806549067, 'val_loss': 0.029655907809734344}
.......................60
60 {'acc': 0.99163362428125845, 'loss': 0.026955342133151216, 'val_acc': 0.98999997806549067, 'val_loss': 0.029523011147975921}
.......................61
61 {'acc': 0.99163362413840883, 'loss': 0.026708778487803723, 'val_acc': 0.98999997806549067, 'val_loss': 0.029289579153060914}
.......................62
62 {'acc': 0.99172077376243362, 'loss': 0.026435853237082245, 'val_acc': 0.98999997806549067, 'val_loss': 0.029149123936891556}
.......................63
63 {'acc': 0.99206937311563026, 'loss': 0.026218523159171871, 'val_acc': 0.98999997806549067, 'val_loss': 0.02900228801369667}
.......................64
64 {'acc': 0.9920257985178923, 'loss': 0.025984287807226305, 'val_acc': 0.98999997806549067, 'val_loss': 0.028836922883987426}
.......................65
65 {'acc': 0.99202579808934355, 'loss': 0.025731697525246153, 'val_acc': 0.98999997806549067, 'val_loss': 0.028719297349452971}
.......................66
66 {'acc': 0.99220009790879149, 'loss': 0.025532959737239142, 'val_acc': 0.98999997806549067, 'val_loss': 0.028700835824012755}
.......................67
67 {'acc': 0.99228724724711703, 'loss': 0.02528472081412118, 'val_acc': 0.98999997806549067, 'val_loss': 0.028460304558277132}
.......................68
68 {'acc': 0.99237439701399144, 'loss': 0.025063294108215697, 'val_acc': 0.98999997806549067, 'val_loss': 0.028242528140544891}
.......................69
69 {'acc': 0.99241797189742864, 'loss': 0.024832777146445939, 'val_acc': 0.98999997806549067, 'val_loss': 0.028136045753955841}
.......................70
70 {'acc': 0.99250512195000218, 'loss': 0.024647444988539591, 'val_acc': 0.99049998474121093, 'val_loss': 0.027945463359355927}
.......................71
71 {'acc': 0.99250512152145343, 'loss': 0.024475379988094671, 'val_acc': 0.98999997806549067, 'val_loss': 0.027918537080287933}
.......................72
72 {'acc': 0.99237439715684106, 'loss': 0.02429658904800883, 'val_acc': 0.98999997806549067, 'val_loss': 0.027792064517736435}
.......................73
73 {'acc': 0.9924615464951666, 'loss': 0.024018169172253294, 'val_acc': 0.99049995565414428, 'val_loss': 0.027615201964974405}
.......................74
74 {'acc': 0.99276657096492615, 'loss': 0.0238396210247039, 'val_acc': 0.99049995565414428, 'val_loss': 0.027539837330579759}
.......................75
75 {'acc': 0.99259227171687658, 'loss': 0.023665886182646263, 'val_acc': 0.98999997806549067, 'val_loss': 0.027502264589071273}
.......................76
76 {'acc': 0.99272299541658893, 'loss': 0.023461740462916953, 'val_acc': 0.99049998474121093, 'val_loss': 0.027263873487710952}
.......................77
77 {'acc': 0.99276657015717651, 'loss': 0.023307317719912134, 'val_acc': 0.98999997806549067, 'val_loss': 0.027312939345836638}
.......................78
78 {'acc': 0.99289729590093689, 'loss': 0.023131982310140575, 'val_acc': 0.99049998474121093, 'val_loss': 0.027106434151530266}
.......................79
79 {'acc': 0.99267942105520213, 'loss': 0.022950362485221771, 'val_acc': 0.99099999141693118, 'val_loss': 0.026936671584844588}
.......................80
80 {'acc': 0.99307159543468559, 'loss': 0.022749435550394159, 'val_acc': 0.99049997806549073, 'val_loss': 0.0271103173494339}
.......................81
81 {'acc': 0.99281014599121287, 'loss': 0.022606046751444779, 'val_acc': 0.99049998474121093, 'val_loss': 0.026812936574220656}
.......................82
82 {'acc': 0.99281014556266411, 'loss': 0.022379495501125415, 'val_acc': 0.98999997806549067, 'val_loss': 0.026826073259115218}
.......................83
83 {'acc': 0.99294087049867485, 'loss': 0.02228978752753559, 'val_acc': 0.99049998474121093, 'val_loss': 0.026603641286492348}
.......................84
84 {'acc': 0.9929408695480757, 'loss': 0.022081843391580282, 'val_acc': 0.99099996232986454, 'val_loss': 0.026502892822027207}
.......................85
85 {'acc': 0.99307159529183597, 'loss': 0.021901483702676618, 'val_acc': 0.99049998474121093, 'val_loss': 0.026545689314603806}
.......................86
86 {'acc': 0.99315874382241198, 'loss': 0.02175932532666041, 'val_acc': 0.99099996232986454, 'val_loss': 0.026447189033031462}
.......................87
87 {'acc': 0.99298444552496168, 'loss': 0.021656327322547492, 'val_acc': 0.99050000715255737, 'val_loss': 0.026460972130298615}
.......................88
88 {'acc': 0.99324589453988554, 'loss': 0.021450444085233649, 'val_acc': 0.99099999141693118, 'val_loss': 0.026221975833177567}
.......................89
89 {'acc': 0.99333304378470955, 'loss': 0.021324197368025832, 'val_acc': 0.99099999141693118, 'val_loss': 0.026163755148649215}
.......................90
90 {'acc': 0.99355091915249472, 'loss': 0.021122420970505529, 'val_acc': 0.99049998474121093, 'val_loss': 0.026192023426294328}
.......................91
91 {'acc': 0.99337661947589628, 'loss': 0.021001940236486424, 'val_acc': 0.99099999141693118, 'val_loss': 0.026028215810656549}
.......................92
92 {'acc': 0.99350734412620789, 'loss': 0.020948878530219903, 'val_acc': 0.99050001382827757, 'val_loss': 0.026064252182841301}
.......................93
93 {'acc': 0.99342019355158395, 'loss': 0.02065966066434545, 'val_acc': 0.99049999141693112, 'val_loss': 0.025869241505861281}
.......................94
94 {'acc': 0.99363806877651939, 'loss': 0.020558752535022474, 'val_acc': 0.99099999141693118, 'val_loss': 0.025964643716812133}
.......................95
95 {'acc': 0.99368164394565583, 'loss': 0.020422524343751563, 'val_acc': 0.99099999141693118, 'val_loss': 0.025924378663301467}
.......................96
96 {'acc': 0.99389951836284174, 'loss': 0.020275591493391167, 'val_acc': 0.99049999141693112, 'val_loss': 0.025691517382860185}
.......................97
97 {'acc': 0.99376879328398138, 'loss': 0.02017625037783153, 'val_acc': 0.99049999141693112, 'val_loss': 0.025591694980859756}
.......................98
98 {'acc': 0.99394309281773019, 'loss': 0.020043422988311927, 'val_acc': 0.99049996232986448, 'val_loss': 0.025540421381592751}
.......................99
99 {'acc': 0.9937252185433938, 'loss': 0.019877716711103395, 'val_acc': 0.99099999141693118, 'val_loss': 0.025714648097753524}

In [ ]:

    




    

In [5]:

    

score=model.evaluate(X_valid,Y_valid, verbose=0)
print("OOS %s: %.2f%%" % (model.metrics_names[1], score[1]*100))
print("OOS %s: %.2f" % (model.metrics_names[0], score[0]))

#print("min(val los)",np.min(history.history['val_loss']))


    

    




OOS acc: 99.10%
OOS loss: 0.03

In [6]:

    

# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='lower right')
plt.show()


# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss (log scale)')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper right')
plt.yscale('log')
plt.show()


    

In [7]:

    

from keras.models import load_model
from sklearn.metrics import log_loss

model = load_model("ResNet-head-Dense2.h5")
with h5py.File("ResNet50-300x300_codes-test.h5") as hf:
     X_test = hf["X_test"][:]


    

In [8]:

    

Y_train_preds=model.predict(X_train)
print(log_loss(Y_train,Y_train_preds))
Y_valid_preds=model.predict(X_valid)
print(log_loss(Y_valid,Y_valid_preds))


    

    




0.0197293185243
0.0257146473853

In [9]:

    

for i in range(1,10):
    m=i/100.0
    print(m,log_loss(Y_valid,Y_valid_preds.clip(min=m, max=1-m)))
    
for i in range(1,10):
    m=i/100.0
    print(m,log_loss(Y_train,Y_train_preds.clip(min=m, max=1-m)))


    

    




0.01 0.0338038361608
0.02 0.0428599127205
0.03 0.0522148580924
0.04 0.0617933657486
0.05 0.0715946517959
0.06 0.0815958315395
0.07 0.0917674249485
0.08 0.102088050611
0.09 0.112511019211
0.01 0.0277817506332
0.02 0.0368027210934
0.03 0.0461583567202
0.04 0.0557684585934
0.05 0.0655785367798
0.06 0.0755731418931
0.07 0.0857064590752
0.08 0.0959931560553
0.09 0.106439248855

In [10]:

    

import re
datagen = ImageDataGenerator()
test_batches = datagen.flow_from_directory("test", model.input_shape[1:3], shuffle=False, batch_size=32, class_mode=None)
#test_codes = model.predict_generator(test_batches, test_batches.nb_sample)

ids = [re.split('/|\.',x)[1] for x in test_batches.filenames]


    

    




Found 12500 images belonging to 1 classes.

In [11]:

    

Y_test = model.predict(X_test)


    

In [12]:

    

Y_test_csv=np.column_stack((ids,Y_test[:,1]))
np.savetxt('Y_test_ResNet-head-Dense2.csv',Y_test_csv, fmt="%s,%s", header=("id,label"),comments='')


    

In [13]:

    

Y_test_csv=np.column_stack((ids,Y_test[:,1].clip(min=0.02, max=0.98)))
np.savetxt('Y_test_ResNet-head-Dense2-clipped.csv',Y_test_csv, fmt="%s,%s", header=("id,label"),comments='')


    

In [ ]: