CLR Callback



In [5]:

    
from clr_callback import *
from keras.optimizers import *

Test Set



In [10]:

    
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Input



In [11]:

    
inp = Input(shape=(15,))                
x = Dense(10, activation='relu')(inp)
x = Dense(1, activation='sigmoid')(x)

model = Model(inp, x)



In [12]:

    
X = np.random.rand(2000000,15)



In [13]:

    
Y = np.random.randint(0,2,size=2000000)

Callback test



In [39]:

    
%matplotlib inline
import matplotlib.pyplot as plt

The following tests callback functionality for each default Keras optimizer.

Tests demonstrate expected cyclical behavior, as well as functionality of resetting the cycles and/or adjusting boundary values.

SGD



In [32]:

    
clr_triangular = CyclicLR(mode='triangular')

model.compile(optimizer=SGD(0.1), loss='binary_crossentropy', metrics=['accuracy'])



In [33]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[33]:





<keras.callbacks.History at 0x7f8f34af34e0>



In [42]:

    
plt.xlabel('Training Iterations')
plt.ylabel('Learning Rate')
plt.title("CLR - 'triangular' Policy")
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[42]:





[<matplotlib.lines.Line2D at 0x7f8f3069ca58>]



In [45]:

    
clr_triangular = CyclicLR(mode='triangular2')

model.compile(optimizer=SGD(), loss='binary_crossentropy', metrics=['accuracy'])



In [46]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[46]:





<keras.callbacks.History at 0x7f8f3bbf4ef0>



In [47]:

    
clr_triangular._reset()



In [48]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[48]:





<keras.callbacks.History at 0x7f8f3bbff908>



In [49]:

    
plt.xlabel('Training Iterations')
plt.ylabel('Learning Rate')
plt.title("'triangular2' Policy Reset at 20000 Iterations")
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[49]:





[<matplotlib.lines.Line2D at 0x7f8f3bbfff28>]



In [25]:

    
clr_triangular = CyclicLR(mode='exp_range', gamma=0.99994)

model.compile(optimizer=SGD(), loss='binary_crossentropy', metrics=['accuracy'])



In [26]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[26]:





<keras.callbacks.History at 0x7fbf6e96b9e8>



In [27]:

    
plt.xlabel('Training Iterations')
plt.ylabel('Learning Rate')
plt.title("CLR - 'exp_range' Policy")
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[27]:





[<matplotlib.lines.Line2D at 0x7fbf6dd3e550>]



In [22]:

    
clr_triangular._reset(new_base_lr=0.003, new_max_lr=0.009)



In [23]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[23]:





<keras.callbacks.History at 0x7fb1b3e44208>



In [24]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[24]:





[<matplotlib.lines.Line2D at 0x7fb1b37e02e8>]



In [28]:

    
clr = lambda x: 0.5*(1+np.sin(x*np.pi/2.))



In [29]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='cycle')

model.compile(optimizer=SGD(), loss='binary_crossentropy', metrics=['accuracy'])



In [30]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=40, callbacks=[clr_triangular], verbose=0)









    Out[30]:





<keras.callbacks.History at 0x7fbf6e142908>



In [31]:

    
plt.xlabel('Training Iterations')
plt.ylabel('Learning Rate')
plt.title("CLR - Custom Sinusoidal Cycle-Policy")
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[31]:





[<matplotlib.lines.Line2D at 0x7fbf6ddd60b8>]



In [50]:

    
clr = lambda x: 1/(5**(x*0.0001))



In [51]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='iterations')

model.compile(optimizer=SGD(), loss='binary_crossentropy', metrics=['accuracy'])



In [52]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[52]:





<keras.callbacks.History at 0x7f8f2fa954a8>



In [53]:

    
plt.xlabel('Training Iterations')
plt.ylabel('Learning Rate')
plt.title("CLR - Custom Iteration-Policy")
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[53]:





[<matplotlib.lines.Line2D at 0x7f8f2fa956d8>]

RMSprop



In [112]:

    
clr_triangular = CyclicLR(mode='triangular')

model.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])



In [113]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[113]:





<keras.callbacks.History at 0x7fcf0088c278>



In [115]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[115]:





[<matplotlib.lines.Line2D at 0x7fcef2275128>]



In [116]:

    
clr_triangular = CyclicLR(mode='triangular2')

model.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])



In [117]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[117]:





<keras.callbacks.History at 0x7fcefa6e87f0>



In [118]:

    
clr_triangular._reset()



In [119]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[119]:





<keras.callbacks.History at 0x7fcefa6e8828>



In [120]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[120]:





[<matplotlib.lines.Line2D at 0x7fcef2a169e8>]



In [121]:

    
clr_triangular = CyclicLR(mode='exp_range', gamma=0.99994)

model.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])



In [122]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[122]:





<keras.callbacks.History at 0x7fcef3825be0>



In [123]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[123]:





[<matplotlib.lines.Line2D at 0x7fcef39234a8>]



In [124]:

    
clr_triangular._reset(new_base_lr=0.003, new_max_lr=0.009)



In [125]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[125]:





<keras.callbacks.History at 0x7fcef26f77f0>



In [126]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[126]:





[<matplotlib.lines.Line2D at 0x7fcef2f9bb38>]



In [127]:

    
clr = lambda x: 0.5*(1+np.sin(x*np.pi/2.))



In [128]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='cycle')

model.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])



In [129]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=40, callbacks=[clr_triangular], verbose=0)









    Out[129]:





<keras.callbacks.History at 0x7fcef3259048>



In [130]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[130]:





[<matplotlib.lines.Line2D at 0x7fcefa3a7c50>]



In [131]:

    
clr = lambda x: 1/(5**(x*0.0001))



In [132]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='iterations')

model.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['accuracy'])



In [133]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[133]:





<keras.callbacks.History at 0x7fcef38d9438>



In [134]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[134]:





[<matplotlib.lines.Line2D at 0x7fcef28554e0>]

Adagrad



In [135]:

    
clr_triangular = CyclicLR(mode='triangular')

model.compile(optimizer=Adagrad(), loss='binary_crossentropy', metrics=['accuracy'])



In [136]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[136]:





<keras.callbacks.History at 0x7fcef232e208>



In [138]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[138]:





[<matplotlib.lines.Line2D at 0x7fcef23d05f8>]



In [139]:

    
clr_triangular = CyclicLR(mode='triangular2')

model.compile(optimizer=Adagrad(), loss='binary_crossentropy', metrics=['accuracy'])



In [140]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[140]:





<keras.callbacks.History at 0x7fcef2d69ba8>



In [141]:

    
clr_triangular._reset()



In [142]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[142]:





<keras.callbacks.History at 0x7fcef2d696a0>



In [143]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[143]:





[<matplotlib.lines.Line2D at 0x7fcefa3094a8>]



In [144]:

    
clr_triangular = CyclicLR(mode='exp_range', gamma=0.99994)

model.compile(optimizer=Adagrad(), loss='binary_crossentropy', metrics=['accuracy'])



In [145]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[145]:





<keras.callbacks.History at 0x7fcefa317208>



In [146]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[146]:





[<matplotlib.lines.Line2D at 0x7fcef309e470>]



In [147]:

    
clr_triangular._reset(new_base_lr=0.003, new_max_lr=0.009)



In [148]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[148]:





<keras.callbacks.History at 0x7fcef2be8a58>



In [149]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[149]:





[<matplotlib.lines.Line2D at 0x7fceffece908>]



In [150]:

    
clr = lambda x: 0.5*(1+np.sin(x*np.pi/2.))



In [151]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='cycle')

model.compile(optimizer=Adagrad(), loss='binary_crossentropy', metrics=['accuracy'])



In [152]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=40, callbacks=[clr_triangular], verbose=0)









    Out[152]:





<keras.callbacks.History at 0x7fcef21e9908>



In [153]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[153]:





[<matplotlib.lines.Line2D at 0x7fcef233a8d0>]



In [154]:

    
clr = lambda x: 1/(5**(x*0.0001))



In [155]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='iterations')

model.compile(optimizer=Adagrad(), loss='binary_crossentropy', metrics=['accuracy'])



In [156]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[156]:





<keras.callbacks.History at 0x7fceffc00320>



In [157]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[157]:





[<matplotlib.lines.Line2D at 0x7fcef31b3518>]

Adadelta



In [158]:

    
clr_triangular = CyclicLR(mode='triangular')

model.compile(optimizer=Adadelta(), loss='binary_crossentropy', metrics=['accuracy'])



In [159]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[159]:





<keras.callbacks.History at 0x7fcef347c3c8>



In [161]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[161]:





[<matplotlib.lines.Line2D at 0x7fcefa63a0f0>]



In [162]:

    
clr_triangular = CyclicLR(mode='triangular2')

model.compile(optimizer=Adadelta(), loss='binary_crossentropy', metrics=['accuracy'])



In [163]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[163]:





<keras.callbacks.History at 0x7fcef43e26d8>



In [164]:

    
clr_triangular._reset()



In [165]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[165]:





<keras.callbacks.History at 0x7fcef273f160>



In [166]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[166]:





[<matplotlib.lines.Line2D at 0x7fcef3f21358>]



In [167]:

    
clr_triangular = CyclicLR(mode='exp_range', gamma=0.99994)

model.compile(optimizer=Adadelta(), loss='binary_crossentropy', metrics=['accuracy'])



In [168]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[168]:





<keras.callbacks.History at 0x7fcef2261a20>



In [169]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[169]:





[<matplotlib.lines.Line2D at 0x7fcefa249fd0>]



In [170]:

    
clr_triangular._reset(new_base_lr=0.003, new_max_lr=0.009)



In [171]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[171]:





<keras.callbacks.History at 0x7fcef2c56748>



In [172]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[172]:





[<matplotlib.lines.Line2D at 0x7fcef34d3e80>]



In [173]:

    
clr = lambda x: 0.5*(1+np.sin(x*np.pi/2.))



In [174]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='cycle')

model.compile(optimizer=Adadelta(), loss='binary_crossentropy', metrics=['accuracy'])



In [175]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=40, callbacks=[clr_triangular], verbose=0)









    Out[175]:





<keras.callbacks.History at 0x7fcefa1eff98>



In [176]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[176]:





[<matplotlib.lines.Line2D at 0x7fcef30ff860>]



In [177]:

    
clr = lambda x: 1/(5**(x*0.0001))



In [178]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='iterations')

model.compile(optimizer=Adadelta(), loss='binary_crossentropy', metrics=['accuracy'])



In [179]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[179]:





<keras.callbacks.History at 0x7fcef2d828d0>



In [180]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[180]:





[<matplotlib.lines.Line2D at 0x7fcef2334748>]

Adam



In [181]:

    
clr_triangular = CyclicLR(mode='triangular')

model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])



In [182]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[182]:





<keras.callbacks.History at 0x7fcef9d0eba8>



In [184]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[184]:





[<matplotlib.lines.Line2D at 0x7fcef0f8f7f0>]



In [185]:

    
clr_triangular = CyclicLR(mode='triangular2')

model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])



In [186]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[186]:





<keras.callbacks.History at 0x7fceefde1f98>



In [187]:

    
clr_triangular._reset()



In [188]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[188]:





<keras.callbacks.History at 0x7fceefe39b00>



In [189]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[189]:





[<matplotlib.lines.Line2D at 0x7fceef9ac160>]



In [190]:

    
clr_triangular = CyclicLR(mode='exp_range', gamma=0.99994)

model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])



In [191]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[191]:





<keras.callbacks.History at 0x7fcef23242b0>



In [192]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[192]:





[<matplotlib.lines.Line2D at 0x7fcef27463c8>]



In [193]:

    
clr_triangular._reset(new_base_lr=0.003, new_max_lr=0.009)



In [194]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[194]:





<keras.callbacks.History at 0x7fcef0f31080>



In [195]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[195]:





[<matplotlib.lines.Line2D at 0x7fcef2a41668>]



In [196]:

    
clr = lambda x: 0.5*(1+np.sin(x*np.pi/2.))



In [197]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='cycle')

model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])



In [198]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=40, callbacks=[clr_triangular], verbose=0)









    Out[198]:





<keras.callbacks.History at 0x7fcef0d0eb00>



In [199]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[199]:





[<matplotlib.lines.Line2D at 0x7fcef226c7f0>]



In [200]:

    
clr = lambda x: 1/(5**(x*0.0001))



In [201]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='iterations')

model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])



In [202]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[202]:





<keras.callbacks.History at 0x7fcefa322160>



In [203]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[203]:





[<matplotlib.lines.Line2D at 0x7fcefa26b390>]

Adamax



In [54]:

    
clr_triangular = CyclicLR(mode='triangular')

model.compile(optimizer=Adamax(), loss='binary_crossentropy', metrics=['accuracy'])



In [55]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[55]:





<keras.callbacks.History at 0x7f8f3bb75cc0>



In [56]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[56]:





[<matplotlib.lines.Line2D at 0x7f8f3c0aeac8>]



In [208]:

    
clr_triangular = CyclicLR(mode='triangular2')

model.compile(optimizer=Adamax(), loss='binary_crossentropy', metrics=['accuracy'])



In [209]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[209]:





<keras.callbacks.History at 0x7fcef2360438>



In [210]:

    
clr_triangular._reset()



In [211]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[211]:





<keras.callbacks.History at 0x7fcef31c6dd8>



In [212]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[212]:





[<matplotlib.lines.Line2D at 0x7fcef3041160>]



In [213]:

    
clr_triangular = CyclicLR(mode='exp_range', gamma=0.99994)

model.compile(optimizer=Adamax(), loss='binary_crossentropy', metrics=['accuracy'])



In [214]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[214]:





<keras.callbacks.History at 0x7fcef0aed898>



In [215]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[215]:





[<matplotlib.lines.Line2D at 0x7fcef3477a90>]



In [216]:

    
clr_triangular._reset(new_base_lr=0.003, new_max_lr=0.009)



In [217]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[217]:





<keras.callbacks.History at 0x7fcef2be6518>



In [218]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[218]:





[<matplotlib.lines.Line2D at 0x7fcf00acce10>]



In [219]:

    
clr = lambda x: 0.5*(1+np.sin(x*np.pi/2.))



In [220]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='cycle')

model.compile(optimizer=Adamax(), loss='binary_crossentropy', metrics=['accuracy'])



In [221]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=40, callbacks=[clr_triangular], verbose=0)









    Out[221]:





<keras.callbacks.History at 0x7fcefa658a58>



In [222]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[222]:





[<matplotlib.lines.Line2D at 0x7fcef28e0e80>]



In [223]:

    
clr = lambda x: 1/(5**(x*0.0001))



In [224]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='iterations')

model.compile(optimizer=Adamax(), loss='binary_crossentropy', metrics=['accuracy'])



In [225]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[225]:





<keras.callbacks.History at 0x7fcef25df208>



In [226]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[226]:





[<matplotlib.lines.Line2D at 0x7fcef222b400>]

Nadam



In [227]:

    
clr_triangular = CyclicLR(mode='triangular')

model.compile(optimizer=Nadam(), loss='binary_crossentropy', metrics=['accuracy'])



In [228]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular],verbose=0)









    Out[228]:





<keras.callbacks.History at 0x7fceefbab128>



In [230]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[230]:





[<matplotlib.lines.Line2D at 0x7fcef9d800f0>]



In [231]:

    
clr_triangular = CyclicLR(mode='triangular2')

model.compile(optimizer=Nadam(), loss='binary_crossentropy', metrics=['accuracy'])



In [232]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[232]:





<keras.callbacks.History at 0x7fcef0fbceb8>



In [233]:

    
clr_triangular._reset()



In [234]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[234]:





<keras.callbacks.History at 0x7fcef10b9080>



In [235]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[235]:





[<matplotlib.lines.Line2D at 0x7fceefb84198>]



In [236]:

    
clr_triangular = CyclicLR(mode='exp_range', gamma=0.99994)

model.compile(optimizer=Nadam(), loss='binary_crossentropy', metrics=['accuracy'])



In [237]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=20, callbacks=[clr_triangular], verbose=0)









    Out[237]:





<keras.callbacks.History at 0x7fceed823588>



In [238]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[238]:





[<matplotlib.lines.Line2D at 0x7fcef0dd9710>]



In [239]:

    
clr_triangular._reset(new_base_lr=0.003, new_max_lr=0.009)



In [240]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[240]:





<keras.callbacks.History at 0x7fceefb5b400>



In [241]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[241]:





[<matplotlib.lines.Line2D at 0x7fcef0e93b38>]



In [242]:

    
clr = lambda x: 0.5*(1+np.sin(x*np.pi/2.))



In [243]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='cycle')

model.compile(optimizer=Nadam(), loss='binary_crossentropy', metrics=['accuracy'])



In [244]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=40, callbacks=[clr_triangular], verbose=0)









    Out[244]:





<keras.callbacks.History at 0x7fcefa47f2b0>



In [245]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[245]:





[<matplotlib.lines.Line2D at 0x7fcefa228588>]



In [246]:

    
clr = lambda x: 1/(5**(x*0.0001))



In [247]:

    
clr_triangular = CyclicLR(scale_fn=clr, scale_mode='iterations')

model.compile(optimizer=Nadam(), loss='binary_crossentropy', metrics=['accuracy'])



In [248]:

    
model.fit(X, Y, batch_size=2000, nb_epoch=10, callbacks=[clr_triangular], verbose=0)









    Out[248]:





<keras.callbacks.History at 0x7fcefa47a160>



In [249]:

    
plt.plot(clr_triangular.history['iterations'], clr_triangular.history['lr'])









    Out[249]:





[<matplotlib.lines.Line2D at 0x7fcef281f0b8>]