Skip to content

Commit 3e8e814

Browse files
author
Yusuke Sugomori
committed
CDBN
1 parent 3012646 commit 3e8e814

File tree

2 files changed

+128
-5
lines changed

2 files changed

+128
-5
lines changed

‎DeepBeliefNets.py

Lines changed: 121 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import numpy
2121
from HiddenLayer import HiddenLayer
2222
from LogisticRegression import LogisticRegression
23-
from RestrictedBoltzmannMachine import RBM
23+
from RestrictedBoltzmannMachine import RBM, CRBM
2424
from utils import *
2525

2626

@@ -143,6 +143,80 @@ def predict(self, x):
143143
out = self.log_layer.predict(layer_input)
144144
return out
145145

146+
'''
147+
DBN w/ continuous-valued inputs (Linear Energy)
148+
149+
'''
150+
class CDBN(DBN):
151+
def __init__(self, input=None, label=None,\
152+
n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2,\
153+
numpy_rng=None):
154+
155+
self.x = input
156+
self.y = label
157+
158+
self.sigmoid_layers = []
159+
self.rbm_layers = []
160+
self.n_layers = len(hidden_layer_sizes) # = len(self.rbm_layers)
161+
162+
if numpy_rng is None:
163+
numpy_rng = numpy.random.RandomState(1234)
164+
165+
166+
assert self.n_layers > 0
167+
168+
169+
# construct multi-layer
170+
for i in xrange(self.n_layers):
171+
# layer_size
172+
if i == 0:
173+
input_size = n_ins
174+
else:
175+
input_size = hidden_layer_sizes[i - 1]
176+
177+
# layer_input
178+
if i == 0:
179+
layer_input = self.x
180+
else:
181+
layer_input = self.sigmoid_layers[-1].sample_h_given_v()
182+
183+
# construct sigmoid_layer
184+
sigmoid_layer = HiddenLayer(input=layer_input,
185+
n_in=input_size,
186+
n_out=hidden_layer_sizes[i],
187+
numpy_rng=numpy_rng,
188+
activation=sigmoid)
189+
self.sigmoid_layers.append(sigmoid_layer)
190+
191+
# construct rbm_layer
192+
if i == 0:
193+
rbm_layer = CRBM(input=layer_input, # continuous-valued inputs
194+
n_visible=input_size,
195+
n_hidden=hidden_layer_sizes[i],
196+
W=sigmoid_layer.W, # W, b are shared
197+
hbias=sigmoid_layer.b)
198+
else:
199+
rbm_layer = RBM(input=layer_input,
200+
n_visible=input_size,
201+
n_hidden=hidden_layer_sizes[i],
202+
W=sigmoid_layer.W, # W, b are shared
203+
hbias=sigmoid_layer.b)
204+
205+
self.rbm_layers.append(rbm_layer)
206+
207+
208+
# layer for output using Logistic Regression
209+
self.log_layer = LogisticRegression(input=self.sigmoid_layers[-1].sample_h_given_v(),
210+
label=self.y,
211+
n_in=hidden_layer_sizes[-1],
212+
n_out=n_outs)
213+
214+
# finetune cost: the negative log likelihood of the logistic regression layer
215+
self.finetune_cost = self.log_layer.negative_log_likelihood()
216+
217+
218+
219+
146220

147221
def test_dbn(pretrain_lr=0.1, pretraining_epochs=1000, k=1, \
148222
finetune_lr=0.1, finetune_epochs=200):
@@ -181,5 +255,51 @@ def test_dbn(pretrain_lr=0.1, pretraining_epochs=1000, k=1, \
181255
print dbn.predict(x)
182256

183257

258+
259+
def test_cdbn(pretrain_lr=0.1, pretraining_epochs=1000, k=1, \
260+
finetune_lr=0.1, finetune_epochs=200):
261+
262+
x = numpy.array([[0.4, 0.5, 0.5, 0., 0., 0.],
263+
[0.5, 0.3, 0.5, 0., 0., 0.],
264+
[0.4, 0.5, 0.5, 0., 0., 0.],
265+
[0., 0., 0.5, 0.3, 0.5, 0.],
266+
[0., 0., 0.5, 0.4, 0.5, 0.],
267+
[0., 0., 0.5, 0.5, 0.5, 0.]])
268+
269+
y = numpy.array([[1, 0],
270+
[1, 0],
271+
[1, 0],
272+
[0, 1],
273+
[0, 1],
274+
[0, 1]])
275+
276+
277+
rng = numpy.random.RandomState(123)
278+
279+
# construct DBN
280+
dbn = CDBN(input=x, label=y, n_ins=6, hidden_layer_sizes=[5, 5], n_outs=2, numpy_rng=rng)
281+
282+
# pre-training (TrainUnsupervisedDBN)
283+
dbn.pretrain(lr=pretrain_lr, k=1, epochs=pretraining_epochs)
284+
285+
# fine-tuning (DBNSupervisedFineTuning)
286+
dbn.finetune(lr=finetune_lr, epochs=finetune_epochs)
287+
288+
289+
# test
290+
x = numpy.array([[0.5, 0.5, 0., 0., 0., 0.],
291+
[0., 0., 0., 0.5, 0.5, 0.],
292+
[0.5, 0.5, 0.5, 0.5, 0.5, 0.]])
293+
294+
295+
print dbn.predict(x)
296+
297+
298+
299+
184300
if __name__ == "__main__":
301+
print >> sys.stderr, 'DBN'
185302
test_dbn()
303+
print >> sys.stderr
304+
print >> sys.stderr, 'CDBN'
305+
test_cdbn()

‎RestrictedBoltzmannMachine.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ def reconstruct(self, v):
141141

142142
'''
143143
RBM w/ continuous-valued inputs (Linear Energy)
144+
144145
'''
145146
class CRBM(RBM):
146147
def propdown(self, h):
@@ -162,7 +163,6 @@ def sample_v_given_h(self, h0_sample):
162163

163164
v1_sample = numpy.log((1 - U * (1 - ep))) / a_h
164165

165-
166166
return [v1_mean, v1_sample]
167167

168168

@@ -214,8 +214,8 @@ def test_crbm(learning_rate=0.1, k=1, training_epochs=1000):
214214
# train
215215
for epoch in xrange(training_epochs):
216216
rbm.contrastive_divergence(lr=learning_rate, k=k)
217-
cost = rbm.get_reconstruction_cross_entropy()
218-
print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
217+
# cost = rbm.get_reconstruction_cross_entropy()
218+
# print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
219219

220220

221221
# test
@@ -226,5 +226,8 @@ def test_crbm(learning_rate=0.1, k=1, training_epochs=1000):
226226

227227

228228
if __name__ == "__main__":
229-
# test_rbm()
229+
print >> sys.stderr, 'RBM'
230+
test_rbm()
231+
print >> sys.stderr
232+
print >> sys.stderr, 'CRBM'
230233
test_crbm()

0 commit comments

Comments
 (0)