10/04/2020 improving the neuralnets .

ipynb - Colaboratory

1 # overfit mlp for the moons dataset
2 from sklearn.datasets import make_moons
3 from tensorflow.keras.models import Sequential
4 from tensorflow.keras.layers import Dense
5 from matplotlib import pyplot
6 # generate 2d classification dataset
7 X, y = make_moons(n_samples=100, noise=0.2, random_state=1)
8 # split into train and test setsi
9 n_train = 30
10 trainX, testX = X[:n_train, :], X[n_train:, :]
11 trainy, testy = y[:n_train], y[n_train:]
12 # define model
13 model = Sequential()
14 model.add(Dense(500, input_dim=2, activation='relu'))
15 model.add(Dense(1, activation='sigmoid'))
16 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
17 # fit model
18 history = model.fit(trainX, trainy, epochs=4000, validation_data=(testX, testy), verbose=2)
19 # evaluate the model
20 _, train_acc = model.evaluate(trainX, trainy, verbose=0)
21 _, test_acc = model.evaluate(testX, testy, verbose=0)
22 print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
23  

This is completly over tting Problems

1  

1 # plot loss learning curves
2 pyplot.subplot(211)
3 pyplot.title('Cross-Entropy Loss', pad=-40)
4 pyplot.plot(history.history['loss'], label='train')
5 pyplot.plot(history.history['val_loss'], label='test')
6 l t l d()
https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 1/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory
6 pyplot.legend()
7 # plot accuracy learning curves
8 pyplot.subplot(212)
9 pyplot.title('Accuracy', pad=-40)
10 pyplot.plot(history.history['accuracy'], label='train')
11 pyplot.plot(history.history['val_accuracy'], label='test')
12 pyplot.legend()
13 pyplot.show()p
14  

1 # mlp overfit on the two circles dataset with activation regularization before activation
2 from sklearn.datasets import make_circles
3 from tensorflow.keras.layers import Dense
4 from tensorflow.keras.models import Sequential
5 from tensorflow.keras.regularizers import l2
6 from tensorflow.keras.layers import Activation
7 from matplotlib import pyplot
8 # generate 2d classification dataset
9 X, y = make_circles(n_samples=100, noise=0.1, random_state=1)
10 # split into train and test
11 n_train = 30
12 trainX, testX = X[:n_train, :], X[n_train:, :]
13 trainy, testy = y[:n_train], y[n_train:]
https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 2/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory
14 # define model
15 model = Sequential()
16 model.add(Dense(500, input_dim=2, activation='linear', activity_regularizer=l2(0.0001)))
17 model.add(Activation('relu'))
18 model.add(Dense(1, activation='sigmoid'))
19 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
20 # fit model
21 history = model.fit(trainX, trainy, validation_data=(testX, testy), epochs=500, verbose=2)
22 # evaluate the model
23 _, train_acc = model.evaluate(trainX, trainy, verbose=0)
24 _, test_acc = model.evaluate(testX, testy, verbose=0)
25 print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
26 # plot loss learning curves
27 pyplot.subplot(211)
28 pyplot.title('Cross-Entropy Loss', pad=-40)
29 pyplot.plot(history.history['loss'], label='train')
30 pyplot.plot(history.history['val_loss'], label='test')
31 pyplot.legend()
32 # plot accuracy learning curves
33 pyplot.subplot(212)
34 pyplot.title('Accuracy', pad=-40)
35 pyplot.plot(history.history['accuracy'], label='train')
36 pyplot.plot(history.history['val_accuracy'], label='test')
37 pyplot.legend()
38 pyplot.show()
39  

1 # mlp overfit on the moons dataset with simple early stopping
2 from sklearn.datasets import make_moons
3 from tensorflow.keras.models import Sequential
4 from tensorflow.keras.layers import Dense
5 from tensorflow.keras.callbacks import EarlyStopping
6 from matplotlib import pyplot
7 # generate 2d classification dataset
8 X, y = make_moons(n_samples=100, noise=0.2, random_state=1)
9 # split into train and test
10 n_train = 30
11 trainX testX = X[:n train :] X[n train: :]
https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 3/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory
11 trainX, testX = X[:n_train, :], X[n_train:, :]
12 trainy, testy = y[:n_train], y[n_train:]
13 # define model
14 model = Sequential()
15 model.add(Dense(500, input_dim=2, activation='relu'))
16 model.add(Dense(1, activation='sigmoid'))
17 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
18 # simple early stopping
19 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1)
20 # fit model
21 history = model.fit(trainX, trainy, validation_data=(testX, testy), epochs=4000, verbose=2, callbacks=[es])
22 # evaluate the model
23 _, train_acc = model.evaluate(trainX, trainy, verbose=0)
24 _, test_acc = model.evaluate(testX, testy, verbose=0)
25 print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
26 # plot loss learning curves
27 pyplot.subplot(211)
28 pyplot.title('Cross-Entropy Loss', pad=-40)
29 pyplot.plot(history.history['loss'], label='train')
30 pyplot.plot(history.history['val_loss'], label='test')
31 pyplot.legend()
32 # plot accuracy learning curves
33 pyplot.subplot(212)
34 pyplot.title('Accuracy', pad=-40)
35 pyplot.plot(history.history['accuracy'], label='train')
36 pyplot.plot(history.history['val_accuracy'], label='test')
37 pyplot.legend()
38 pyplot.show()

1 # mlp for the two circles problem with batchnorm after activation function
2 from sklearn.datasets import make_circles
3 from tensorflow.keras.models import Sequential
4 from tensorflow.keras.layers import Dense
5 from tensorflow.keras.layers import BatchNormalization
6 from tensorflow.keras.optimizers import SGD
7 from matplotlib import pyplot
8 # generate 2d classification dataset
9 X, y = make_circles(n_samples=1000, noise=0.1, random_state=1)
https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 4/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory

10 # split into train and test
11 n_train = 500
12 trainX, testX = X[:n_train, :], X[n_train:, :]
13 trainy, testy = y[:n_train], y[n_train:]
14 # define model
15 model = Sequential()
16 model.add(Dense(50, input_dim=2, activation='relu', kernel_initializer='he_uniform'))
17 model.add(BatchNormalization())
18 model.add(Dense(1, activation='sigmoid'))
19 opt = SGD(lr=0.01, momentum=0.9)
20 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
21 # fit model
22 history = model.fit(trainX, trainy, validation_data=(testX, testy), epochs=100, verbose=2)
23 # evaluate the model
24 _, train_acc = model.evaluate(trainX, trainy, verbose=0)
25 _, test_acc = model.evaluate(testX, testy, verbose=0)
26 print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
27 # plot loss learning curves
28 pyplot.subplot(211)
29 pyplot.title('Cross-Entropy Loss', pad=-40)
30 pyplot.plot(history.history['loss'], label='train')
31 pyplot.plot(history.history['val_loss'], label='test')
32 pyplot.legend()
33 # plot accuracy learning curves
34 pyplot.subplot(212)
35 pyplot.title('Accuracy', pad=-40)
36 pyplot.plot(history.history['accuracy'], label='train')
37 pyplot.plot(history.history['val_accuracy'], label='test')
38 pyplot.legend()
39 pyplot.show()

Batch normalization will helps in over tting

Drop out

https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 5/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory

1 # mlp overfit on the two circles dataset
2 from sklearn.datasets import make_circles
3 from tensorflow.keras.layers import Dense
4 from tensorflow.keras.models import Sequential
5 from matplotlib import pyplot
6 # generate 2d classification dataset
7 X, y = make_circles(n_samples=100, noise=0.1, random_state=1)
8 # split into train and test
9 n_train = 30
10 trainX, testX = X[:n_train, :], X[n_train:, :]
11 trainy, testy = y[:n_train], y[n_train:]
12 # define model
13 model = Sequential()
14 model.add(Dense(500, input_dim=2, activation='relu'))
15 model.add(Dense(1, activation='sigmoid'))
16 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
17 # fit model
18 history = model.fit(trainX, trainy, validation_data=(testX, testy), epochs=1000, verbose=2)
19 # evaluate the model
20 _, train_acc = model.evaluate(trainX, trainy, verbose=0)
21 _, test_acc = model.evaluate(testX, testy, verbose=0)
22 print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
23 # plot loss learning curves
24 pyplot.subplot(211)
25 pyplot.title('Cross-Entropy Loss', pad=-40)
26 pyplot.plot(history.history['loss'], label='train')
27 pyplot.plot(history.history['val_loss'], label='test')
28 pyplot.legend()
29 # plot accuracy learning curves
30 pyplot.subplot(212)
31 pyplot.title('Accuracy', pad=-40)
32 pyplot.plot(history.history['accuracy'], label='train')
33 pyplot.plot(history.history['val_accuracy'], label='test')
34 pyplot.legend()
35 pyplot.show()

1 model.summary()
https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 6/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory

Model: "sequential_6"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_12 (Dense) (None, 500) 1500
_________________________________________________________________
dense_13 (Dense) (None, 1) 501
=================================================================
Total params: 2,001
Trainable params: 2,001
Non-trainable params: 0
_________________________________________________________________

1 # mlp with dropout on the two circles dataset
2 from sklearn.datasets import make_circles
3 from tensorflow.keras.models import Sequential
a
4 from tensorflow.keras.layers import Dense
a
5 from tensorflow.keras.layers import Dropout
6  
7 from matplotlib import pyplot
8 # generate 2d classification dataset
9 X, y = make_circles(n_samples=100, noise=0.1, random_state=1)
10 # split into train and test
11 n_train = 30
12 trainX, testX = X[:n_train, :], X[n_train:, :]
13 trainy, testy = y[:n_train], y[n_train:]
14 # define model
15 model = Sequential()
16 model.add(Dense(500, input_dim=2, activation='relu'))
17 model.add(Dropout(0.3)) # dropping of 40% of conncections
18 model.add(Dense(1, activation='sigmoid'))
19 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
20 # fit model
21 history = model.fit(trainX, trainy, validation_data=(testX, testy), epochs=1000, verbose=2)
22 # evaluate the model
23 _, train_acc = model.evaluate(trainX, trainy, verbose=0)
24 _, test_acc = model.evaluate(testX, testy, verbose=0)
25 print('Train: % 3f Test: % 3f' % (train acc test acc))
https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 7/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory
25 print( Train: %.3f, Test: %.3f  % (train_acc, test_acc))
26 # plot loss learning curves
27 pyplot.subplot(211)
28 pyplot.title('Cross-Entropy Loss', pad=-40)
29 pyplot.plot(history.history['loss'], label='train')
30 pyplot.plot(history.history['val_loss'], label='test')
31 pyplot.legend()
32 # plot accuracy learning curves
33 pyplot.subplot(212)
34 pyplot.title('Accuracy', pad=-40)
35 pyplot.plot(history.history['accuracy'], label='train')
36 pyplot.plot(history.history['val_accuracy'], label='test')
37 pyplot.legend()
38 pyplot.show()

1 model.summary()

Model: "sequential_7"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_14 (Dense) (None, 500) 1500
_________________________________________________________________
dropout (Dropout) (None, 500) 0
_________________________________________________________________
dense_15 (Dense) (None, 1) 501
=================================================================
Total params: 2,001
Trainable params: 2,001
Non-trainable params: 0
_________________________________________________________________

https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 8/9
10/04/2020 improving the neuralnets .ipynb - Colaboratory

https://colab.research.google.com/drive/1_j9s58sxbknVMY3Nz0aYyjmGqBBEPRbz#scrollTo=EG4zUN6XDVUH&printMode=true 9/9