Logistic Regression

Yusuke Sugomori · Yusuke Sugomori · commit 771c89aa530a · 2013-01-02T22:43:37.000+09:00
diff --git a/DeepBeliefNets.py b/DeepBeliefNets.py
@@ -1,6 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+'''
+ Deep Belief Nets
+'''
+
 import numpy
 import LogisticRegression
 import RestrictedBoltzmannMachine
@@ -9,13 +13,35 @@
 
 
 
-class DeepBeliefNets(object):
-    def __init__(self):
-        print 'DeepBeliefNets'
+class DBN(object):
+    def __init__(self, n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2, \
+                 numpy_rng=None):   # constructor does not contain input
+
+        self.sigmoid_layers = []
+        self.rbm_layers = []
+        self.n_layers = len(hidden_layer_sizes)
+
+
+        if numpy_rng is None:
+            numpy_rng = numpy.random.RandomState(1234)
+        
+
+        assert self.n_layers > 0
+
+
+        # construct multi layers
+        for i in xrange(self.n_layers):
+            if i == 0:
+                input_size = n_ins
+            else:
+                input_size = hidden_layer_sizes[i - 1]
+
 
 
 
+def test_dbn():
+    dbn = DBN()
 
 
 if __name__ == "__main__":
-    print 'DeepBeliefNets'
+    test_dbn()
diff --git a/LogisticRegression.py b/LogisticRegression.py
@@ -1,13 +1,91 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+'''
+ Logistic Regression
+'''
+
+import sys
 import numpy
+from utils import *
 
 
 class LogisticRegression(object):
-    def __init__(self):
-        print 'LogisticRegression'
+    def __init__(self, input=None, n_in=2, n_out=2):
+        self.input = input
+        self.W = numpy.zeros((n_in, n_out))  # initialize W 0
+        self.b = numpy.zeros(n_out)          # initialize bias 0
+
+    def train(self, y, lr=0.1):
+        y_pred = softmax(numpy.dot(self.input, self.W) + self.b)
+        d_y = y - y_pred
+
+        self.W += lr * numpy.dot(self.input.T, d_y)
+        self.b += lr * numpy.mean(d_y, axis=0)
+
+        # cost = self.negative_log_likelihood()
+        # return cost
+
+    def negative_log_likelihood(self):
+        sigmoid_activation = sigmoid(numpy.dot(self.input, self.W) + self.b)
+        
+        # entropy = - numpy.mean(numpy.sum(self.input * numpy.log(sigmoid_activation), axis=1))
+        # return entropy
+        
+        cross_entropy = - numpy.mean(
+            numpy.sum(self.input * numpy.log(sigmoid_activation) +
+            (1 - self.input) * numpy.log(1 - sigmoid_activation),
+                      axis=1))
+
+        return cross_entropy
+
+
+    def predict(self, x):
+        return sigmoid(numpy.dot(x, self.W) + self.b)
+
+
+def test_lr(learning_rate=0.1, n_epochs=100):
+    n_epochs = 10
+
+
+    # training data
+    rng = numpy.random.RandomState(123)
+    n_each = 10
+    m1 = -5.
+    s1 = 1.0
+    m2 = 10.
+    s2 = 10.
+
+    x = []
+    y = []
+    
+    for i in xrange(n_each):
+        x.append([rng.normal(m1, s1), rng.normal(m1, s1)])
+        y.append([0])
+
+    for i in xrange(n_each):
+        x.append([rng.normal(m2, s2), rng.normal(m2, s2)])
+        y.append([1])
+    
+    x = numpy.array(x)
+    y = numpy.array(y)
+
+
+
+    # construct LogisticRegression
+    classifier = LogisticRegression(input=x, n_in=2, n_out=1)
+
+    # train
+    for epoch in xrange(n_epochs):
+        classifier.train(y=y, lr=learning_rate)
+        # cost = classifier.negative_log_likelihood()
+        # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
+
+
+    # test
+    x = numpy.array([-5.0, 1])
+    print >> sys.stderr, classifier.predict(x)
 
 
 if __name__ == "__main__":
-    print 'LogisticRegression'
+    test_lr()
diff --git a/RestrictedBoltzmannMachine.py b/RestrictedBoltzmannMachine.py
@@ -20,7 +20,7 @@
 import numpy
 from utils import *
 
-class RestrictedBoltzmannMachine(object):
+class RBM(object):
     def __init__(self, input=None, n_visible=2, n_hidden=3, \
         W=None, hbias=None, vbias=None, numpy_rng=None):
         
@@ -53,7 +53,7 @@ def __init__(self, input=None, n_visible=2, n_hidden=3, \
         self.hbias = hbias
         self.vbias = vbias
 
-        self.params = [self.W, self.hbias, self.vbias]
+        # self.params = [self.W, self.hbias, self.vbias]
 
 
     def contrastive_divergence(self, lr=0.1, k=1):
@@ -77,8 +77,8 @@ def contrastive_divergence(self, lr=0.1, k=1):
         self.vbias += lr * numpy.mean(self.input - nv_samples, axis=1)
 
 
-        cost = self.get_reconstruction_cross_entropy()
-        return cost
+        # cost = self.get_reconstruction_cross_entropy()
+        # return cost
 
 
     def sample_h_given_v(self, v0_sample):
@@ -154,10 +154,11 @@ def test_rbm(learning_rate=0.1, k=1, training_epochs=15):
     rng = numpy.random.RandomState(123)
 
     # construct RBM
-    rbm = RestrictedBoltzmannMachine(input=data, n_visible=6, n_hidden=2, numpy_rng=rng)
+    rbm = RBM(input=data, n_visible=6, n_hidden=2, numpy_rng=rng)
 
     for epoch in xrange(training_epochs):
-        cost = rbm.contrastive_divergence(lr=learning_rate, k=k)
+        rbm.contrastive_divergence(lr=learning_rate, k=k)
+        cost = rbm.get_reconstruction_cross_entropy()
         print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
 
 
diff --git a/utils.py b/utils.py
@@ -1,5 +1,20 @@
 ''' '''
 import numpy
 
+numpy.seterr(all='ignore')
+
+
 def sigmoid(x):
+    # numpy.seterrcall(sigmoid_err_handler)  # overflow handling
+    # numpy.seterr(all='call')
     return 1. / (1 + numpy.exp(-x))
+
+
+def sigmoid_err_handler(type, flg):
+    # Log
+    return
+
+
+def softmax(x):
+    e = numpy.exp(x - numpy.max(x, axis=0))  # prevent overflow
+    return e / numpy.sum(e)
