Activation Function

Sigmoid



In [48]:

    
import numpy as np
x = np.linspace(-10, 10, 1000)
y = 1 / (1 + np.exp(-1 * x))
y_dx = np.exp(-1 * x) / (1 + np.exp(-1 * x))**2



In [49]:

    
%matplotlib inline
import matplotlib.pyplot as plt

plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(x, y)
plt.title('Sigmoid Function', fontsize=15)
plt.subplot(1, 2, 2)
plt.plot(x, y_dx)
plt.title('Derivative of Sigmoid', fontsize=15)
plt.show()

tanh (Hyperbolic Tangent Function)



In [50]:

    
import numpy as np
x = np.linspace(-10, 10, 1000)
y = (np.exp(x) - np.exp(-1 * x)) / (np.exp(x) + np.exp(-1*x))
y_dx = 1 - y**2



In [51]:

    
%matplotlib inline
import matplotlib.pyplot as plt

plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(x, y)
plt.title('tanh Function', fontsize=15)
plt.subplot(1, 2, 2)
plt.plot(x, y_dx)
plt.title('Derivative of tanh', fontsize=15)
plt.show()

ReLU



In [52]:

    
import numpy as np
x = np.linspace(-10, 10, 1000)
y = np.where(x>0, x, 0)
y_dx = np.where(x>0, 1, 0)



In [53]:

    
%matplotlib inline
import matplotlib.pyplot as plt

plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(x, y)
plt.title('ReLU Function', fontsize=15)
plt.subplot(1, 2, 2)
plt.plot(x, y_dx)
plt.title('Derivative of ReLU', fontsize=15)
plt.show()

Leaky ReLU



In [54]:

    
import numpy as np
x = np.linspace(-10, 10, 1000)
y = np.where(x>0.01*x, x, 0.01*x)
y_dx = np.where(x>0.01*x, 1, 0.01)



In [55]:

    
%matplotlib inline
import matplotlib.pyplot as plt

plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(x, y)
plt.title('Leaky ReLU Function', fontsize=15)
plt.subplot(1, 2, 2)
plt.plot(x, y_dx)
plt.title('Derivative of Leaky ReLU', fontsize=15)
plt.show()

ELU



In [56]:

    
import numpy as np
x = np.linspace(-10, 10, 1000)
alpha = 0.1
y = np.where(x>0, x, alpha*(np.exp(x)-1))
y_dx = np.where(x>0, 1, alpha*np.exp(x))



In [57]:

    
%matplotlib inline
import matplotlib.pyplot as plt

plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
plt.plot(x, y)
plt.title('ELU Function', fontsize=15)
plt.subplot(1, 2, 2)
plt.plot(x, y_dx)
plt.title('Derivative of ELU', fontsize=15)
plt.show()