激活函数的实现

激活函数的实现

Sigmoid函数的实现

class Sigmoid(Operator):
    def __init__(self, input_variable=Variable, name=str):
        self.input_variables = input_variable
        self.output_variables = Variable(self.input_variables.shape,
         name='out', scope=name)
        Operator.__init__(self, name, self.input_variables, 
        self.output_variables)

    def forward(self):
        if self.wait_forward:
            for parent in self.parent:
                GLOBAL_VARIABLE_SCOPE[parent].eval()
            # y = 1/(1+exp(-x))
            self.output_variables.data = 1.0/(1.0+
            np.exp(-self.input_variables.data))
            self.wait_forward = False
            return
        else:
            pass

    def backward(self):
        if self.wait_forward:
            pass
        else:
            for child in self.child:
                GLOBAL_VARIABLE_SCOPE[child].diff_eval()
            # eta_x = eta_y * (1-y) * y
            self.input_variables.diff = self.output_variables.data * (
            1 - self.output_variables.data) * self.output_variables.diff
            self.wait_forward = True
            return

Relu函数

forward:
    self.output_variables.data = np.maximum(self.input_variables.data, 0)
backward:
    self.output_variables.diff[self.input_variables.data < 0] = 0

总结

import numpy as np


class ReluActivator(object):
    def forward(self, weighted_input):
        return max(0, weighted_input)

    def backward(self, output):
        return 1 if output > 0 else 0


class IdentityActivator(object):
    def forward(self, weighted_input):
        return weighted_input

    def backward(self, output):
        return 1


class SigmoidActivator(object):
    def forward(self, weighted_input):
        return 1.0 / (1.0 + np.exp(-weighted_input))
    def backward(self, output):
        return output * (1 - output)


class TanhActivator(object):
    def forward(self, weighted_input):
        return 2.0 / (1.0 + np.exp(-2 * weighted_input)) - 1.0

    def backward(self, output):
        return 1 - output * output