Skip to content

Commit c4765a9

Browse files
author
Yusuke Sugomori
committed
SdA
1 parent 6c7c743 commit c4765a9

File tree

1 file changed

+184
-0
lines changed

1 file changed

+184
-0
lines changed

‎SdA.py

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
4+
5+
"""
6+
Stacked Denoising Autoencoders (SdA)
7+
8+
References :
9+
- P. Vincent, H. Larochelle, Y. Bengio, P.A. Manzagol: Extracting and
10+
Composing Robust Features with Denoising Autoencoders, ICML' 08, 1096-1103,
11+
2008
12+
13+
- DeepLearningTutorials
14+
https://github.com/lisa-lab/DeepLearningTutorials
15+
16+
"""
17+
18+
import sys
19+
import numpy
20+
from HiddenLayer import HiddenLayer
21+
from LogisticRegression import LogisticRegression
22+
from dA import dA
23+
from utils import *
24+
25+
26+
class SdA(object):
27+
def __init__(self, input=None, label=None,\
28+
n_ins=2, hidden_layer_sizes=[3, 3], n_outs=2,\
29+
numpy_rng=None):
30+
31+
self.x = input
32+
self.y = label
33+
34+
self.sigmoid_layers = []
35+
self.dA_layers = []
36+
self.n_layers = len(hidden_layer_sizes) # = len(self.rbm_layers)
37+
38+
if numpy_rng is None:
39+
numpy_rng = numpy.random.RandomState(1234)
40+
41+
42+
assert self.n_layers > 0
43+
44+
45+
# construct multi-layer
46+
for i in xrange(self.n_layers):
47+
# layer_size
48+
if i == 0:
49+
input_size = n_ins
50+
else:
51+
input_size = hidden_layer_sizes[i - 1]
52+
53+
# layer_input
54+
if i == 0:
55+
layer_input = self.x
56+
else:
57+
layer_input = self.sigmoid_layers[-1].sample_h_given_v()
58+
59+
# construct sigmoid_layer
60+
sigmoid_layer = HiddenLayer(input=layer_input,
61+
n_in=input_size,
62+
n_out=hidden_layer_sizes[i],
63+
numpy_rng=numpy_rng,
64+
activation=sigmoid)
65+
self.sigmoid_layers.append(sigmoid_layer)
66+
67+
68+
# construct rbm_layer
69+
# rbm_layer = RBM(input=layer_input,
70+
# n_visible=input_size,
71+
# n_hidden=hidden_layer_sizes[i],
72+
# W=sigmoid_layer.W, # W, b are shared
73+
# hbias=sigmoid_layer.b)
74+
# self.rbm_layers.append(rbm_layer)
75+
76+
# construct dA_layers
77+
dA_layer = dA(input=layer_input,
78+
n_visible=input_size,
79+
n_hidden=hidden_layer_sizes[i],
80+
W=sigmoid_layer.W,
81+
hbias=sigmoid_layer.b)
82+
self.dA_layers.append(dA_layer)
83+
84+
85+
# layer for output using Logistic Regression
86+
self.log_layer = LogisticRegression(input=self.sigmoid_layers[-1].sample_h_given_v(),
87+
label=self.y,
88+
n_in=hidden_layer_sizes[-1],
89+
n_out=n_outs)
90+
91+
# finetune cost: the negative log likelihood of the logistic regression layer
92+
self.finetune_cost = self.log_layer.negative_log_likelihood()
93+
94+
95+
def pretrain(self, lr=0.1, corruption_level=0.3, epochs=100):
96+
for i in xrange(self.n_layers):
97+
if i == 0:
98+
layer_input = self.x
99+
else:
100+
layer_input = self.sigmoid_layers[i-1].sample_h_given_v(layer_input)
101+
102+
da = self.dA_layers[i]
103+
104+
for epoch in xrange(epochs):
105+
da.train(lr=lr, corruption_level=corruption_level, input=layer_input)
106+
107+
def finetune(self, lr=0.1, epochs=100):
108+
layer_input = self.sigmoid_layers[-1].sample_h_given_v()
109+
110+
# train log_layer
111+
epoch = 0
112+
113+
while epoch < epochs:
114+
self.log_layer.train(lr=lr, input=layer_input)
115+
# self.finetune_cost = self.log_layer.negative_log_likelihood()
116+
# print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, self.finetune_cost
117+
118+
lr *= 0.95
119+
epoch += 1
120+
121+
122+
def predict(self, x):
123+
layer_input = x
124+
125+
for i in xrange(self.n_layers):
126+
sigmoid_layer = self.sigmoid_layers[i]
127+
# rbm_layer = self.rbm_layers[i]
128+
layer_input = sigmoid_layer.output(input=layer_input)
129+
130+
out = self.log_layer.predict(layer_input)
131+
return out
132+
133+
134+
135+
def test_SdA(pretrain_lr=0.1, pretraining_epochs=1000, corruption_level=0.3, \
136+
finetune_lr=0.1, finetune_epochs=200):
137+
x = numpy.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
138+
[1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
139+
[1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
140+
[1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
141+
[0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
142+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
143+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1],
144+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1],
145+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1],
146+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0]])
147+
148+
y = numpy.array([[1, 0],
149+
[1, 0],
150+
[1, 0],
151+
[1, 0],
152+
[1, 0],
153+
[0, 1],
154+
[0, 1],
155+
[0, 1],
156+
[0, 1],
157+
[0, 1]])
158+
159+
160+
rng = numpy.random.RandomState(123)
161+
162+
# construct SdA
163+
sda = SdA(input=x, label=y, \
164+
n_ins=20, hidden_layer_sizes=[10, 10], n_outs=2, numpy_rng=rng)
165+
166+
# pre-training
167+
sda.pretrain(lr=pretrain_lr, corruption_level=corruption_level, epochs=pretraining_epochs)
168+
169+
# fine-tuning
170+
sda.finetune(lr=finetune_lr, epochs=finetune_epochs)
171+
172+
173+
# test
174+
x = numpy.array([[1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
175+
[1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
176+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1],
177+
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1]])
178+
179+
print sda.predict(x)
180+
181+
182+
183+
if __name__ == "__main__":
184+
test_SdA()

0 commit comments

Comments
 (0)