Skip to content

Commit 5b73c27

Browse files
author
Yusuke Sugomori
committed
RBM
1 parent 890bb96 commit 5b73c27

File tree

3 files changed

+247
-18
lines changed

3 files changed

+247
-18
lines changed

‎cpp/LogisticRegression.cpp

+14-18
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ LogisticRegression::LogisticRegression(int size, int in, int out) {
1111
n_out = out;
1212

1313
// initialize W, b
14-
W = new double*[n_in];
15-
for (int i=0; i<n_in; i++) W[i] = new double[n_out];
14+
W = new double*[n_out];
15+
for (int i=0; i<n_out; i++) W[i] = new double[n_in];
1616
b = new double[n_out];
1717
}
1818

@@ -24,22 +24,21 @@ LogisticRegression::~LogisticRegression() {
2424

2525

2626
void LogisticRegression::train(int *x, int *y, double lr) {
27-
int i,j;
2827
double p_y_given_x[n_out];
2928
double dy[n_out];
3029

31-
for (i=0; i<n_out; i++) {
32-
for (j=0; j<n_in; j++) {
30+
for (int i=0; i<n_out; i++) {
31+
for (int j=0; j<n_in; j++) {
3332
p_y_given_x[i] += W[i][j] * x[j];
3433
}
3534
p_y_given_x[i] += b[i];
3635
}
3736
softmax(p_y_given_x);
3837

39-
for (i=0; i<n_out; i++) {
38+
for (int i=0; i<n_out; i++) {
4039
dy[i] = y[i] - p_y_given_x[i];
4140

42-
for (j=0; j<n_in; j++) {
41+
for (int j=0; j<n_in; j++) {
4342
W[i][j] += lr * dy[i] * x[j] / N;
4443
}
4544

@@ -48,17 +47,16 @@ void LogisticRegression::train(int *x, int *y, double lr) {
4847
}
4948

5049
void LogisticRegression::softmax(double *x) {
51-
double max;
52-
double sum;
50+
double max = 0.0;
51+
double sum = 0.0;
5352

54-
int i;
55-
for (i=0; i<n_out; i++) if(max < x[i]) max = x[i];
56-
for (i=0; i<n_out; i++) {
53+
for (int i=0; i<n_out; i++) if(max < x[i]) max = x[i];
54+
for (int i=0; i<n_out; i++) {
5755
x[i] = exp(x[i] - max);
5856
sum += x[i];
5957
}
6058

61-
for(i=0; i<n_out; i++) x[i] /= sum;
59+
for(int i=0; i<n_out; i++) x[i] /= sum;
6260
}
6361

6462
void LogisticRegression::predict(int *x, double *y) {
@@ -74,8 +72,6 @@ void LogisticRegression::predict(int *x, double *y) {
7472

7573

7674
void test_lr() {
77-
int i,j;
78-
7975
double learning_rate = 0.1;
8076
double n_epochs = 500;
8177

@@ -129,7 +125,7 @@ void test_lr() {
129125

130126
// train online
131127
for (int epoch=0; epoch<n_epochs; epoch++) {
132-
for (i=0; i<train_N; i++) {
128+
for (int i=0; i<train_N; i++) {
133129
classifier.train(train_X[i], train_Y[i], learning_rate);
134130
}
135131
learning_rate *= 0.95;
@@ -145,9 +141,9 @@ void test_lr() {
145141

146142

147143
// test
148-
for (i=0; i<test_N; i++) {
144+
for (int i=0; i<test_N; i++) {
149145
classifier.predict(test_X[i], test_Y[i]);
150-
for (j=0; j<n_out; j++) {
146+
for (int j=0; j<n_out; j++) {
151147
cout << test_Y[i][j] << endl;
152148
}
153149
}

‎cpp/RBM.cpp

+210
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
#include <iostream>
2+
#include <math.h>
3+
#include "RBM.h"
4+
using namespace std;
5+
6+
7+
RBM::RBM(int size, int n_v, int n_h) {
8+
N = size;
9+
n_visible = n_v;
10+
n_hidden = n_h;
11+
12+
W = new double*[n_hidden];
13+
for (int i=0; i<n_hidden; i++) W[i] = new double[n_visible];
14+
hbias = new double[n_hidden];
15+
vbias = new double[n_visible];
16+
17+
double a = 1.0 / n_visible;
18+
19+
for(int i=0; i<n_hidden; i++) {
20+
for(int j=0; j<n_visible; j++) {
21+
W[i][j] = uniform(-a, a);
22+
}
23+
}
24+
}
25+
26+
RBM::~RBM() {
27+
for (int i=0; i<n_hidden; i++) delete[] W[i];
28+
delete[] W;
29+
delete[] hbias;
30+
delete[] vbias;
31+
}
32+
33+
double RBM::uniform(double min, double max) {
34+
return rand() / (RAND_MAX + 1.0) * (max - min) + min;
35+
}
36+
37+
int RBM::binomial(int n, double p) {
38+
if(p < 0 || p > 1) return 0;
39+
40+
int c = 0;
41+
double r;
42+
43+
for (int i=0; i<n; i++) {
44+
r = rand() / (RAND_MAX + 1.0);
45+
if (r < p) c++;
46+
}
47+
48+
return c;
49+
}
50+
51+
double RBM::sigmoid(double x) {
52+
return 1.0 / (1.0 + exp(-x));
53+
}
54+
55+
56+
void RBM::contrastive_divergence(int *input, double lr, int k) {
57+
double *ph_mean = new double[n_hidden];
58+
int *ph_sample = new int[n_hidden];
59+
double *nv_means = new double[n_visible];
60+
int *nv_samples = new int[n_visible];
61+
double *nh_means = new double[n_hidden];
62+
int *nh_samples = new int[n_hidden];
63+
64+
/* CD-k */
65+
sample_h_given_v(input, ph_mean, ph_sample);
66+
67+
for (int step=0; step<k; step++) {
68+
if (step == 0) {
69+
gibbs_hvh(ph_sample, nv_means, nv_samples, nh_means, nh_samples);
70+
} else {
71+
gibbs_hvh(nh_samples, nv_means, nv_samples, nh_means, nh_samples);
72+
}
73+
}
74+
75+
for (int i=0; i<n_hidden; i++) {
76+
for (int j=0; j<n_visible; j++) {
77+
W[i][j] += lr * (ph_sample[i] * input[j] - nh_means[i] * nv_samples[j]);
78+
}
79+
hbias[i] += lr * (ph_sample[i] - nh_means[i]);
80+
}
81+
82+
for (int i=0; i<n_visible; i++) {
83+
vbias[i] += lr * (input[i] - nv_samples[i]);
84+
}
85+
86+
delete[] ph_mean;
87+
delete[] ph_sample;
88+
delete[] nv_means;
89+
delete[] nv_samples;
90+
delete[] nh_means;
91+
delete[] nh_samples;
92+
}
93+
94+
void RBM::sample_h_given_v(int *v0_sample, double *mean, int *sample) {
95+
for (int i=0; i<n_hidden; i++) {
96+
mean[i] = propup(v0_sample, W[i], hbias[i]);
97+
sample[i] = binomial(1, mean[i]);
98+
}
99+
}
100+
101+
void RBM::sample_v_given_h(int *h0_sample, double *mean, int *sample) {
102+
for (int i=0; i<n_visible; i++) {
103+
mean[i] = propdown(h0_sample, i, vbias[i]);
104+
sample[i] = binomial(1, mean[i]);
105+
}
106+
}
107+
108+
double RBM::propup(int *v, double *w, double b) {
109+
double pre_sigmoid_activation = 0.0;
110+
for(int j=0; j<n_visible; j++) {
111+
pre_sigmoid_activation += w[j] * v[j];
112+
}
113+
pre_sigmoid_activation += b;
114+
return sigmoid(pre_sigmoid_activation);
115+
}
116+
117+
double RBM::propdown(int *h, int i, double b) {
118+
double pre_sigmoid_activation = 0.0;
119+
for (int j=0; j<n_hidden; j++) {
120+
pre_sigmoid_activation += W[j][i] * h[j];
121+
}
122+
pre_sigmoid_activation += b;
123+
return sigmoid(pre_sigmoid_activation);
124+
}
125+
126+
void RBM::gibbs_hvh(int *h0_sample, double *nv_means, int *nv_samples, \
127+
double *nh_means, int *nh_samples) {
128+
sample_v_given_h(h0_sample, nv_means, nv_samples);
129+
sample_h_given_v(nv_samples, nh_means, nh_samples);
130+
}
131+
132+
void RBM::reconstruct(int *v, double *reconstructed_v) {
133+
double *h = new double[n_hidden];
134+
double pre_sigmoid_activation;
135+
136+
for (int i=0; i<n_hidden; i++) {
137+
h[i] = propup(v, W[i], hbias[i]);
138+
}
139+
140+
for (int i=0; i<n_visible; i++) {
141+
pre_sigmoid_activation = 0.0;
142+
for (int j=0; j<n_hidden; j++) {
143+
pre_sigmoid_activation += W[j][i] * h[j];
144+
}
145+
pre_sigmoid_activation += vbias[i];
146+
147+
reconstructed_v[i] = sigmoid(pre_sigmoid_activation);
148+
}
149+
150+
delete[] h;
151+
}
152+
153+
154+
void test_rbm() {
155+
srand(0);
156+
157+
double learning_rate = 0.1;
158+
int training_epochs = 100;
159+
int k = 1;
160+
161+
int train_N = 6;
162+
int test_N = 2;
163+
int n_visible = 6;
164+
int n_hidden = 3;
165+
166+
// training data
167+
int train_X[6][6] = {
168+
{1, 1, 1, 0, 0, 0},
169+
{1, 0, 1, 0, 0, 0},
170+
{1, 1, 1, 0, 0, 0},
171+
{0, 0, 1, 1, 1, 0},
172+
{0, 0, 1, 0, 1, 0},
173+
{0, 0, 1, 1, 1, 0}
174+
};
175+
176+
// construct RBM
177+
RBM rbm(train_N, n_visible, n_hidden);
178+
179+
// train
180+
for (int epoch=0; epoch<training_epochs; epoch++) {
181+
for (int i=0; i<train_N; i++) {
182+
rbm.contrastive_divergence(train_X[i], learning_rate, k);
183+
}
184+
}
185+
186+
// test data
187+
int test_X[2][6] = {
188+
{1, 1, 0, 0, 0, 0},
189+
{0, 0, 0, 1, 1, 0}
190+
};
191+
double reconstructed_X[2][6];
192+
193+
194+
// test
195+
for (int i=0; i<test_N; i++) {
196+
rbm.reconstruct(test_X[i], reconstructed_X[i]);
197+
for (int j=0; j<n_visible; j++) {
198+
printf("%.5f ", reconstructed_X[i][j]);
199+
}
200+
cout << endl;
201+
}
202+
203+
}
204+
205+
206+
207+
int main() {
208+
test_rbm();
209+
return 0;
210+
}

‎cpp/RBM.h

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
class RBM {
2+
3+
public:
4+
int N;
5+
int n_visible;
6+
int n_hidden;
7+
double **W;
8+
double *hbias;
9+
double *vbias;
10+
RBM(int, int, int);
11+
~RBM();
12+
double uniform(double, double);
13+
int binomial(int, double);
14+
double sigmoid(double);
15+
void contrastive_divergence(int*, double, int);
16+
void sample_h_given_v(int*, double*, int*);
17+
void sample_v_given_h(int*, double*, int*);
18+
double propup(int*, double*, double);
19+
double propdown(int*, int, double);
20+
void gibbs_hvh(int*, double*, int*, double*, int*);
21+
void reconstruct(int*, double*);
22+
};
23+

0 commit comments

Comments
 (0)