Added A General Swish Activation Function inNeural Networks

neural_network/activation_functions/sigmoid_linear_unit.py → neural_network/activation_functions/swish.py

@@ -12,6 +12,7 @@
 
 This script is inspired by a corresponding research paper.
 * https://arxiv.org/abs/1710.05941
+* https://blog.paperspace.com/swish-activation-function/
 """
 
 import numpy as np
@@ -49,6 +50,25 @@ def sigmoid_linear_unit(vector: np.ndarray) -> np.ndarray:
     return vector * sigmoid(vector)
 
 
+def swish(vector: np.ndarray, trainable_parameter: int) -> np.ndarray:
+    """
+    Parameters:
+        vector (np.ndarray): A  numpy array consisting of real values
+        trainable_parameter: Use to implement various Swish Activation Functions
+
+    Returns:
+        swish_vec (np.ndarray): The input numpy array, after applying swish
+
+    Examples:
+    >>> swish(np.array([-1.0, 1.0, 2.0]), 2)
+    array([-0.11920292,  0.88079708,  1.96402758])
+
+    >>> swish(np.array([-2]), 1)
+    array([-0.23840584])
+    """
+    return vector * sigmoid(trainable_parameter * vector)
+
+
 if __name__ == "__main__":
     import doctest