import numpy as np

# Collection of activation functions

# Reference: https://en.wikipedia.org/wiki/Activation_function

class Sigmoid():

    def __call__(self, x):

        return 1 / (1 + np.exp(-x))

    def gradient(self, x):

        return self.__call__(x) * (1 - self.__call__(x))

class Softmax():

    def __call__(self, x):

        e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))

        return e_x / np.sum(e_x, axis=-1, keepdims=True)

    def gradient(self, x):

        p = self.__call__(x)

        return p * (1 - p)

class TanH():

    def __call__(self, x):

        return 2 / (1 + np.exp(-2*x)) - 1

    def gradient(self, x):

        return 1 - np.power(self.__call__(x), 2)

class ReLU():

    def __call__(self, x):

        return np.where(x >= 0, x, 0)

    def gradient(self, x):

        return np.where(x >= 0, 1, 0)

class LeakyReLU():

    def __init__(self, alpha=0.2):

        self.alpha = alpha

    def __call__(self, x):

        return np.where(x >= 0, x, self.alpha * x)

    def gradient(self, x):

        return np.where(x >= 0, 1, self.alpha)

class ELU():

    def __init__(self, alpha=0.1):

        self.alpha = alpha 

    def __call__(self, x):

        return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))

    def gradient(self, x):

        return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)

class SELU():

    # Reference : https://arxiv.org/abs/1706.02515,

    # https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb

    def __init__(self):

        self.alpha = 1.6732632423543772848170429916717

        self.scale = 1.0507009873554804934193349852946 

    def __call__(self, x):

        return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1))

    def gradient(self, x):

        return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))

class SoftPlus():

    def __call__(self, x):

        return np.log(1 + np.exp(x))

    def gradient(self, x):

        return 1 / (1 + np.exp(-x))