Skip to content

Commit 48a3d5c

Browse files
author
Yusuke Sugomori
committed
DBN
1 parent 5e4d0b0 commit 48a3d5c

File tree

3 files changed

+90
-108
lines changed

3 files changed

+90
-108
lines changed

‎DeepBeliefNets.py

Lines changed: 38 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,20 @@
1414

1515

1616
class DBN(object):
17-
def __init__(self, input=None, n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2, \
17+
def __init__(self, input=None, label=None, n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2, \
1818
numpy_rng=None): # constructor does not contain input
1919

20-
self.input = input
20+
self.x = input
21+
self.y = label
2122

2223
self.sigmoid_layers = []
2324
self.rbm_layers = []
2425
self.n_layers = len(hidden_layer_sizes) # = len(self.rbm_layers)
2526

2627
if numpy_rng is None:
2728
numpy_rng = numpy.random.RandomState(1234)
28-
2929

30+
3031
assert self.n_layers > 0
3132

3233

@@ -40,7 +41,7 @@ def __init__(self, input=None, n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2, \
4041

4142
# layer_input
4243
if i == 0:
43-
layer_input = self.input
44+
layer_input = self.x
4445
else:
4546
layer_input = self.sigmoid_layers[-1].output()
4647

@@ -62,28 +63,22 @@ def __init__(self, input=None, n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2, \
6263
self.rbm_layers.append(rbm_layer)
6364

6465

65-
# exit()
66-
67-
6866
# layer for output using Logistic Regression
6967
self.log_layer = LogisticRegression(input=self.sigmoid_layers[-1].output(),
68+
label=self.y,
7069
n_in=hidden_layer_sizes[-1],
7170
n_out=n_outs)
7271

7372
# finetune cost: the negative log likelihood of the logistic regression layer
74-
# self.finetune_cost = self.log_layer.negative_log_likelihood()
75-
# print self.finetune_cost
76-
77-
78-
# self.errors
73+
self.finetune_cost = self.log_layer.negative_log_likelihood()
7974

8075

8176

8277
def pretrain(self, lr=0.1, k=1, epochs=100):
8378
# pre-train layer-wise
8479
for i in xrange(self.n_layers):
8580
if i == 0:
86-
layer_input = self.input
81+
layer_input = self.x
8782
else:
8883
layer_input = self.sigmoid_layers[i-1].output()
8984
rbm = self.rbm_layers[i]
@@ -92,92 +87,74 @@ def pretrain(self, lr=0.1, k=1, epochs=100):
9287
for epoch in xrange(epochs):
9388
c = []
9489
rbm.contrastive_divergence(lr=lr, k=k, input=layer_input)
95-
cost = rbm.get_reconstruction_cross_entropy()
96-
# c.append(cost)
97-
print >> sys.stderr, \
98-
'Pre-training layer %d, epoch %d, cost ' %(i, epoch), cost
90+
# cost = rbm.get_reconstruction_cross_entropy()
91+
# # c.append(cost)
92+
# print >> sys.stderr, \
93+
# 'Pre-training layer %d, epoch %d, cost ' %(i, epoch), cost
9994

10095
# print numpy.mean(c)
10196

10297

103-
def finetune(self, y, lr=0.1, epochs=100):
98+
def finetune(self, lr=0.1, epochs=100):
10499
layer_input = self.sigmoid_layers[-1].output()
105100

106101
# train log_layer
107102
epoch = 0
108103
done_looping = False
109104
while (epoch < epochs) and (not done_looping):
110-
self.log_layer.train(y=y, lr=lr, input=layer_input)
111-
cost = self.log_layer.negative_log_likelihood(y=y)
112-
print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
105+
self.log_layer.train(lr=lr, input=layer_input)
106+
# self.finetune_cost = self.log_layer.negative_log_likelihood()
107+
# print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, self.finetune_cost
113108

114109
lr *= 0.95
115110
epoch += 1
116111

117-
# print self.log_layer.W
118-
# exit()
119112

120113
def predict(self, x):
121114
layer_input = x
122115

123116
for i in xrange(self.n_layers):
124-
print layer_input
125117
sigmoid_layer = self.sigmoid_layers[i]
118+
# rbm_layer = self.rbm_layers[i]
126119
layer_input = sigmoid_layer.output(input=layer_input)
127120

128-
# print self.log_layer.W
129121
out = self.log_layer.predict(layer_input)
130-
print layer_input
131-
132122
return out
133123

134124

135-
def test_dbn(pretrain_lr=0.1, pretraining_epochs=10, k=1, \
136-
finetune_lr=0.1, finetune_epochs=10):
137-
138-
x = numpy.array([[0, 0, 0, 1],
139-
[0, 0, 1, 0],
140-
[0, 0, 1, 0],
141-
[0, 1, 0, 0],
142-
[0, 1, 0, 0],
143-
[1, 0, 0, 0]])
144-
y = numpy.array([[0, 0, 0, 1],
145-
[0, 0, 1, 0],
146-
[0, 0, 1, 0],
147-
[0, 1, 0, 0],
148-
[0, 1, 0, 0],
149-
[1, 0, 0, 0]])
125+
def test_dbn(pretrain_lr=0.1, pretraining_epochs=1000, k=1, \
126+
finetune_lr=0.1, finetune_epochs=1000):
127+
128+
x = numpy.array([[1,1,1,0,0,0],
129+
[1,0,1,0,0,0],
130+
[1,1,1,0,0,0],
131+
[0,0,1,1,1,0],
132+
[0,0,1,1,0,0],
133+
[0,0,1,1,1,0]])
134+
y = numpy.array([[1, 0],
135+
[1, 0],
136+
[1, 0],
137+
[0, 1],
138+
[0, 1],
139+
[0, 1]])
140+
150141

151142
rng = numpy.random.RandomState(123)
152143

153144
# construct DBN
154-
dbn = DBN(input=x, n_ins=4, hidden_layer_sizes=[10], n_outs=4, numpy_rng=rng)
145+
dbn = DBN(input=x, label=y, n_ins=6, hidden_layer_sizes=[2], n_outs=2, numpy_rng=rng)
155146

156147
# pre-training (TrainUnsupervisedDBN)
157148
dbn.pretrain(lr=pretrain_lr, k=1, epochs=pretraining_epochs)
158149

159150
# fine-tuning (DBNSupervisedFineTuning)
160-
dbn.finetune(y, lr=finetune_lr, epochs=finetune_epochs)
151+
dbn.finetune(lr=finetune_lr, epochs=finetune_epochs)
161152

162153

163154
# test
164-
x = numpy.array([0, 0, 0, 1])
165-
print dbn.predict(x) # 0
166-
print
167-
168-
169-
exit()
170-
x = numpy.array([0, 0])
171-
print dbn.predict(x) # 0
172-
print
173-
x = numpy.array([0, 1])
174-
print dbn.predict(x) # 1
175-
print
176-
x = numpy.array([1, 0])
177-
print dbn.predict(x) # 1
178-
print
179-
x = numpy.array([1, 1])
180-
print dbn.predict(x) # 0
155+
x = numpy.array([1, 1, 0, 0, 0, 0])
156+
print dbn.predict(x)
157+
181158

182159
if __name__ == "__main__":
183160
test_dbn()

‎LogisticRegression.py

Lines changed: 33 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -17,75 +17,73 @@
1717

1818

1919
class LogisticRegression(object):
20-
def __init__(self, input, n_in, n_out):
21-
self.input = input
20+
def __init__(self, input, label, n_in, n_out):
21+
self.x = input
22+
self.y = label
2223
self.W = numpy.zeros((n_in, n_out)) # initialize W 0
2324
self.b = numpy.zeros(n_out) # initialize bias 0
2425

2526
# self.params = [self.W, self.b]
2627

27-
def train(self, y, lr=0.1, input=None):
28+
def train(self, lr=0.1, input=None):
2829
if input is not None:
29-
self.input = input
30+
self.x = input
3031

31-
p_y_given_x = softmax(numpy.dot(self.input, self.W) + self.b)
32-
d_y = y - p_y_given_x
32+
p_y_given_x = softmax(numpy.dot(self.x, self.W) + self.b)
33+
d_y = self.y - p_y_given_x
3334

34-
self.W += lr * numpy.dot(self.input.T, d_y)
35+
self.W += lr * numpy.dot(self.x.T, d_y)
3536
self.b += lr * numpy.mean(d_y, axis=0)
36-
3737

3838
# cost = self.negative_log_likelihood()
3939
# return cost
4040

41-
def negative_log_likelihood(self, y):
42-
sigmoid_activation = softmax(numpy.dot(self.input, self.W) + self.b)
41+
def negative_log_likelihood(self):
42+
sigmoid_activation = softmax(numpy.dot(self.x, self.W) + self.b)
4343

44-
# entropy = - numpy.mean(numpy.sum(y * numpy.log(sigmoid_activation), axis=1))
45-
# return entropy
46-
4744
cross_entropy = - numpy.mean(
48-
numpy.sum(y * numpy.log(sigmoid_activation) +
49-
(1 - y) * numpy.log(1 - sigmoid_activation),
45+
numpy.sum(self.y * numpy.log(sigmoid_activation) +
46+
(1 - self.y) * numpy.log(1 - sigmoid_activation),
5047
axis=1))
5148

5249
return cross_entropy
5350

5451

5552
def predict(self, x):
56-
return sigmoid(numpy.dot(x, self.W) + self.b)
53+
return softmax(numpy.dot(x, self.W) + self.b)
5754

5855

59-
def test_lr(learning_rate=0.01, n_epochs=5):
56+
def test_lr(learning_rate=0.01, n_epochs=1000):
6057
# training data
61-
x = numpy.array([[-10., -5.],
62-
[-5., -10.],
63-
[5., 10.],
64-
[10., 5.]])
65-
y = numpy.array([[0, 0, 0, 0],
66-
[0, 0, 0, 0],
67-
[1, 1, 1, 1],
68-
[1, 1, 1, 1]])
58+
x = numpy.array([[1,1,1,0,0,0],
59+
[1,0,1,0,0,0],
60+
[1,1,1,0,0,0],
61+
[0,0,1,1,1,0],
62+
[0,0,1,1,0,0],
63+
[0,0,1,1,1,0]])
64+
y = numpy.array([[1, 0],
65+
[1, 0],
66+
[1, 0],
67+
[0, 1],
68+
[0, 1],
69+
[0, 1]])
6970

7071

7172
# construct LogisticRegression
72-
classifier = LogisticRegression(input=x, n_in=2, n_out=4)
73+
classifier = LogisticRegression(input=x, label=y, n_in=6, n_out=2)
7374

7475
# train
7576
for epoch in xrange(n_epochs):
76-
classifier.train(y=y, lr=learning_rate)
77-
cost = classifier.negative_log_likelihood(y=y)
78-
print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
77+
classifier.train(lr=learning_rate)
78+
# cost = classifier.negative_log_likelihood(y=y)
79+
# print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
7980
learning_rate *= 0.95
8081

8182

8283
# test
83-
print
84-
print 'test'
85-
x = numpy.array([-10., -5.])
86-
print >> sys.stderr, classifier.predict(x) # 0
87-
x = numpy.array([10., 5.])
88-
print >> sys.stderr, classifier.predict(x) # 1
84+
x = numpy.array([1, 1, 0, 0, 0, 0])
85+
print >> sys.stderr, classifier.predict(x)
86+
8987

9088
if __name__ == "__main__":
9189
test_lr()

‎RestrictedBoltzmannMachine.py

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,9 @@ def contrastive_divergence(self, lr=0.1, k=1, input=None):
7777

7878

7979
self.W += lr * (numpy.dot(self.input.T, ph_sample)
80-
- numpy.dot(nv_samples.T, nh_samples))
81-
self.hbias += lr * numpy.mean(ph_sample - nh_samples, axis=0)
80+
- numpy.dot(nv_samples.T, nh_means))
8281
self.vbias += lr * numpy.mean(self.input - nv_samples, axis=0)
83-
82+
self.hbias += lr * numpy.mean(ph_sample - nh_means, axis=0)
8483

8584
# cost = self.get_reconstruction_cross_entropy()
8685
# return cost
@@ -138,27 +137,35 @@ def get_reconstruction_cross_entropy(self):
138137

139138

140139

141-
def test_rbm(learning_rate=0.1, k=1, training_epochs=20):
142-
data = numpy.array([
143-
[1,1,1,0,0],
144-
[1,0,1,0,0],
145-
[1,1,1,0,0],
146-
[0,0,1,1,1],
147-
[0,0,1,1,0],
148-
[0,0,1,1,1]])
140+
def test_rbm(learning_rate=0.1, k=1, training_epochs=1000):
141+
data = numpy.array([[1,1,1,0,0,0],
142+
[1,0,1,0,0,0],
143+
[1,1,1,0,0,0],
144+
[0,0,1,1,1,0],
145+
[0,0,1,1,0,0],
146+
[0,0,1,1,1,0]])
147+
149148

150149

151150
rng = numpy.random.RandomState(123)
152151

153152
# construct RBM
154-
rbm = RBM(input=data, n_visible=5, n_hidden=2, numpy_rng=rng)
153+
rbm = RBM(input=data, n_visible=6, n_hidden=2, numpy_rng=rng)
155154

156155
for epoch in xrange(training_epochs):
157156
rbm.contrastive_divergence(lr=learning_rate, k=k)
158157
cost = rbm.get_reconstruction_cross_entropy()
159158
print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
160159

161160

161+
# test
162+
# v = numpy.array([0, 0, 0, 1, 1, 0])
163+
v = numpy.array([1, 1, 0, 0, 0, 0])
164+
h = sigmoid(numpy.dot(v, rbm.W) + rbm.hbias)
165+
166+
reconstructed_v = sigmoid(numpy.dot(h, rbm.W.T) + rbm.vbias)
167+
print reconstructed_v
168+
162169

163170
if __name__ == "__main__":
164171
test_rbm()

0 commit comments

Comments
 (0)