Dirivatives of Activation Functions
Created by Sanasam Ranbir Singh
In [5]:
import numpy as np 1. Derivative of Sigmoid Activation Function
In [8]:
# I am creating a Sigmoid function
def sigmoid(x):
return 1/(1 + np.exp(-x))
# I am creating derivative of Sigmoid function
def dSigmoid(x):
return sigmoid(x)*(1-sigmoid(x))In [10]:
# Use the above function to estimate scores of different values of x
x = 1.0
print('Sigmoid score of (%.1f): %.2f' % (x, dSigmoid(x)))
x = 0.0
print('Sigmoid score of (%.1f): %.2f' % (x, dSigmoid(x)))
x = -1.0
print('Sigmoid score of (%.1f): %.2f' % (x, dSigmoid(x)))
Sigmoid score of (1.0): 0.20 Sigmoid score of (0.0): 0.25 Sigmoid score of (-1.0): 0.20
In [15]:
# Plot distribution of sigmoid scores over diffrent values of x [-ve to +ve]
import matplotlib.pyplot as plt # you would need to implement matplotlib "pip install matplotlib"
x = np.linspace(-10, 10, 51) # returns number spaces evenly w.r.t interval 25 -ve evenly spaced values and 25 +ve evenly spaced values.
print(x)
# find sigmoid values of the elements in the vector x
val = sigmoid(x) # val is also a vector: val[i] is the sigmoid value of x[i]
dVal = dSigmoid(x) # val is also a vector: val[i] is the sigmoid value of x[i]
plt.xlabel("x")
plt.ylabel("Sigmoid(x)")
plt.plot(x, val)
plt.plot(x, dVal)
plt.legend(["sigmoid", "derivative"])
plt.grid()
plt.show()
[-10. -9.6 -9.2 -8.8 -8.4 -8. -7.6 -7.2 -6.8 -6.4 -6. -5.6 -5.2 -4.8 -4.4 -4. -3.6 -3.2 -2.8 -2.4 -2. -1.6 -1.2 -0.8 -0.4 0. 0.4 0.8 1.2 1.6 2. 2.4 2.8 3.2 3.6 4. 4.4 4.8 5.2 5.6 6. 6.4 6.8 7.2 7.6 8. 8.4 8.8 9.2 9.6 10. ]
2. tanh Activation function
Python Numpy has "tanh" function i.e., np.tanh()
In [23]:
def dTanh(x):
return (1-np.tanh(x)*np.tanh(x))In [24]:
x = 1.0
print('derivative tanh score of (%.1f): %.1f' % (x, dTanh(x)))
x = 0.0
print('derivative tanh score of (%.1f): %.1f' % (x, dTanh(x)))
x = -1.0
print('derivative tanh score of (%.1f): %.1f' % (x, dTanh(x)))derivative tanh score of (1.0): 0.4 derivative tanh score of (0.0): 1.0 derivative tanh score of (-1.0): 0.4
In [25]:
# Plot distribution of sigmoid scores over diffrent values of x [-ve to +ve]
import matplotlib.pyplot as plt # you would need to implement matplotlib "pip install matplotlib"
x = np.linspace(-10, 10, 51) # returns number spaces evenly w.r.t interval 25 -ve evenly spaced values and 25 +ve evenly spaced values.
print(x)
# find tanh values of the elements in the vector x
val = np.tanh(x) # val is also a vector: val[i] is the tanh value of x[i]
dVal = dTanh(x)
plt.xlabel("x")
plt.ylabel("tanh(x)")
plt.plot(x, val)
plt.plot(x, dVal)
plt.legend(["tanh", "derivative"])
plt.grid()
plt.show()[-10. -9.6 -9.2 -8.8 -8.4 -8. -7.6 -7.2 -6.8 -6.4 -6. -5.6 -5.2 -4.8 -4.4 -4. -3.6 -3.2 -2.8 -2.4 -2. -1.6 -1.2 -0.8 -0.4 0. 0.4 0.8 1.2 1.6 2. 2.4 2.8 3.2 3.6 4. 4.4 4.8 5.2 5.6 6. 6.4 6.8 7.2 7.6 8. 8.4 8.8 9.2 9.6 10. ]
3. Derivatives ReLu activation function
In [34]:
def relu(x):
return np.maximum(0.0,x)
def dRelu(x):
tp = [1 if val > 0 else 0 for val in x]
return np.array(tp)
In [40]:
import matplotlib.pyplot as plt
x = np.linspace(-10, 10, 51)
print(x)
# find tanh values of the elements in the vector x
val = relu(x) # val is also a vector: val[i] is the tanh value of x[i]
dVal = dRelu(x)
plt.xlabel("x")
plt.ylabel("ReLU(x)")
plt.plot(x, val)
plt.plot(x, dVal)
plt.legend(["relu", "derivative"])
plt.grid()
plt.show()[-10. -9.6 -9.2 -8.8 -8.4 -8. -7.6 -7.2 -6.8 -6.4 -6. -5.6 -5.2 -4.8 -4.4 -4. -3.6 -3.2 -2.8 -2.4 -2. -1.6 -1.2 -0.8 -0.4 0. 0.4 0.8 1.2 1.6 2. 2.4 2.8 3.2 3.6 4. 4.4 4.8 5.2 5.6 6. 6.4 6.8 7.2 7.6 8. 8.4 8.8 9.2 9.6 10. ]
4. Derivative of Leaky ReLu activation function
In [41]:
def leaky_relu(x):
return np.maximum(0.05*x,x)
def dLRelu(x):
tp = [0.05 if val < 0 else 1 for val in x]
return np.array(tp)In [44]:
import matplotlib.pyplot as plt
x = np.linspace(-10, 10, 51)
print(x)
# find tanh values of the elements in the vector x
val = leaky_relu(x) # val is also a vector: val[i] is the tanh value of x[i]
dVal = dLRelu(x)
plt.xlabel("x")
plt.ylabel("Leaky ReLU(x)")
plt.plot(x, val)
plt.plot(x, dVal)
plt.legend(["leaky relu", "derivative"])
plt.grid()
plt.show()[-10. -9.6 -9.2 -8.8 -8.4 -8. -7.6 -7.2 -6.8 -6.4 -6. -5.6 -5.2 -4.8 -4.4 -4. -3.6 -3.2 -2.8 -2.4 -2. -1.6 -1.2 -0.8 -0.4 0. 0.4 0.8 1.2 1.6 2. 2.4 2.8 3.2 3.6 4. 4.4 4.8 5.2 5.6 6. 6.4 6.8 7.2 7.6 8. 8.4 8.8 9.2 9.6 10. ]
In [ ]: