Sample code

ActivationFunctions.ipynb

Download .ipynb

Different Activation Functions

Created by Sanasam Ranbir Singh

In [4]:
import numpy as np

1. Sigmoid Activation Function

image.png

In [5]:
# I am creating a Sigmoid function
def sigmoid(x):
    return 1/(1 + np.exp(-x))
In [10]:
# Use the above function to estimate scores of different values of x
x = 1.0
print('Sigmoid score of (%.1f): %.1f' % (x, sigmoid(x)))

x = 0.0
print('Sigmoid score of (%.1f): %.1f' % (x, sigmoid(x)))

x = -1.0
print('Sigmoid score of (%.1f): %.1f' % (x, sigmoid(x)))

Sigmoid score of (1.0): 0.7
Sigmoid score of (0.0): 0.5
Sigmoid score of (-1.0): 0.3
In [17]:
# Plot distribution of sigmoid scores over diffrent values of x [-ve to +ve]
import matplotlib.pyplot as plt    # you would need to implement matplotlib "pip install matplotlib"
x = np.linspace(-10, 10, 51)       # returns number spaces evenly w.r.t interval 25 -ve evenly spaced values and 25 +ve evenly spaced values.
print(x)

# find sigmoid values of the elements in the vector x
val = sigmoid(x)            # val is also a vector: val[i] is the sigmoid value of x[i]
plt.xlabel("x") 
plt.ylabel("Sigmoid(x)")  
plt.plot(x, val) 
plt.grid()
plt.show()

[-10.   -9.6  -9.2  -8.8  -8.4  -8.   -7.6  -7.2  -6.8  -6.4  -6.   -5.6
  -5.2  -4.8  -4.4  -4.   -3.6  -3.2  -2.8  -2.4  -2.   -1.6  -1.2  -0.8
  -0.4   0.    0.4   0.8   1.2   1.6   2.    2.4   2.8   3.2   3.6   4.
   4.4   4.8   5.2   5.6   6.    6.4   6.8   7.2   7.6   8.    8.4   8.8
   9.2   9.6  10. ]

2. tanh Activation function

Python Numpy has "tanh" function i.e., np.tanh()

In [20]:
x = 1.0
print('tanh score of (%.1f): %.1f' % (x, np.tanh(x)))

x = 0.0
print('tanh score of (%.1f): %.1f' % (x, np.tanh(x)))

x = -1.0
print('tanh score of (%.1f): %.1f' % (x, np.tanh(x)))
tanh score of (1.0): 0.8
tanh score of (0.0): 0.0
tanh score of (-1.0): -0.8
In [21]:
# Plot distribution of sigmoid scores over diffrent values of x [-ve to +ve]
import matplotlib.pyplot as plt    # you would need to implement matplotlib "pip install matplotlib"
x = np.linspace(-10, 10, 51)       # returns number spaces evenly w.r.t interval 25 -ve evenly spaced values and 25 +ve evenly spaced values.
print(x)

# find tanh values of the elements in the vector x
val = np.tanh(x)            # val is also a vector: val[i] is the tanh value of x[i]
plt.xlabel("x") 
plt.ylabel("tanh(x)")  
plt.plot(x, val) 
plt.grid()
plt.show()
[-10.   -9.6  -9.2  -8.8  -8.4  -8.   -7.6  -7.2  -6.8  -6.4  -6.   -5.6
  -5.2  -4.8  -4.4  -4.   -3.6  -3.2  -2.8  -2.4  -2.   -1.6  -1.2  -0.8
  -0.4   0.    0.4   0.8   1.2   1.6   2.    2.4   2.8   3.2   3.6   4.
   4.4   4.8   5.2   5.6   6.    6.4   6.8   7.2   7.6   8.    8.4   8.8
   9.2   9.6  10. ]

3. ReLu activation function

image.png

In [36]:
def relu(x):
    return np.maximum(0.0,x)
In [37]:
x = 1.0
print('ReLU score of (%.1f): %.1f' % (x, relu(x)))

x = 0.0
print('ReLU score of (%.1f): %.1f' % (x, relu(x)))

x = -1.0
print('ReLU score of (%.1f): %.1f' % (x, relu(x)))
ReLU score of (1.0): 1.0
ReLU score of (0.0): 0.0
ReLU score of (-1.0): 0.0
In [38]:
import matplotlib.pyplot as plt    
x = np.linspace(-10, 10, 51)       
print(x)

# find tanh values of the elements in the vector x
val = relu(x)            # val is also a vector: val[i] is the tanh value of x[i]
plt.xlabel("x") 
plt.ylabel("ReLU(x)")  
plt.plot(x, val) 
plt.grid()
plt.show()
[-10.   -9.6  -9.2  -8.8  -8.4  -8.   -7.6  -7.2  -6.8  -6.4  -6.   -5.6
  -5.2  -4.8  -4.4  -4.   -3.6  -3.2  -2.8  -2.4  -2.   -1.6  -1.2  -0.8
  -0.4   0.    0.4   0.8   1.2   1.6   2.    2.4   2.8   3.2   3.6   4.
   4.4   4.8   5.2   5.6   6.    6.4   6.8   7.2   7.6   8.    8.4   8.8
   9.2   9.6  10. ]

4. ReLu activation function

image.png

In [47]:
def leaky_relu(x):
    return np.maximum(0.05*x,x)
In [48]:
x = 1.0
print('Leaky ReLU score of (%.2f): %.2f' % (x, leaky_relu(x)))

x = 0.0
print('Leaky ReLU score of (%.2f): %.2f' % (x, leaky_relu(x)))

x = -1.0
print('Leaky ReLU score of (%.2f): %.2f' % (x, leaky_relu(x)))
Leaky ReLU score of (1.00): 1.00
Leaky ReLU score of (0.00): 0.00
Leaky ReLU score of (-1.00): -0.05
In [49]:
import matplotlib.pyplot as plt    
x = np.linspace(-10, 10, 51)       
print(x)

# find tanh values of the elements in the vector x
val = leaky_relu(x)            # val is also a vector: val[i] is the tanh value of x[i]
plt.xlabel("x") 
plt.ylabel("Leaky_ReLU(x)")  
plt.plot(x, val) 
plt.grid()
plt.show()
[-10.   -9.6  -9.2  -8.8  -8.4  -8.   -7.6  -7.2  -6.8  -6.4  -6.   -5.6
  -5.2  -4.8  -4.4  -4.   -3.6  -3.2  -2.8  -2.4  -2.   -1.6  -1.2  -0.8
  -0.4   0.    0.4   0.8   1.2   1.6   2.    2.4   2.8   3.2   3.6   4.
   4.4   4.8   5.2   5.6   6.    6.4   6.8   7.2   7.6   8.    8.4   8.8
   9.2   9.6  10. ]
In [ ]: