diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b6797577 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea/ +*.pkl +tutorials/data +*__pycache__* diff --git a/.gitingore b/.gitingore deleted file mode 100644 index a42991c5..00000000 --- a/.gitingore +++ /dev/null @@ -1 +0,0 @@ -Untitled.ipynb diff --git a/my-tutorials/cifar100.py b/my-tutorials/cifar100.py new file mode 100644 index 00000000..5029f1bc --- /dev/null +++ b/my-tutorials/cifar100.py @@ -0,0 +1,78 @@ +""" +CIFAR100 dataset playground, url: https://www.cs.toronto.edu/~kriz/cifar.html + +Couldn't now figure out labels mapping. some images showed wrong labels, +like keyboard was labeled as kangaroo, and cups and crocodile had to be swapped +""" +import time +import torch +import torchvision +import torch.nn as nn +import numpy as np +import torch.utils.data as data +import torchvision.transforms as transforms +import torchvision.datasets as dsets +from torch.autograd import Variable +import matplotlib.pyplot as plt + +from common.pytorch_helpers import DATA_PATH + + +LABELS = [ + ("aquatic mammals", "beaver, dolphin, otter, seal, whale"), + ("fish", "aquarium fish, flatfish, ray, shark, trout"), + ("flowers", "orchids, poppies, roses, sunflowers, tulips"), + ("food containers", "bottles, bowls, cans, cups, plates"), + ("fruit and vegetables", "apples, mushrooms, oranges, pears, sweet peppers"), + ("household electrical devices", "clock, computer keyboard, lamp, telephone, television"), + ("household furniture", "bed, chair, couch, table, wardrobe"), + ("insects", "bee, beetle, butterfly, caterpillar, cockroach"), + ("large carnivores", "bear, leopard, lion, tiger, wolf"), + ("large man-made outdoor things", "bridge, castle, house, road, skyscraper"), + ("large natural outdoor scenes", "cloud, forest, mountain, plain, sea"), + ("large omnivores and herbivores", "camel, cattle, chimpanzee, elephant, kangaroo"), + ("medium-sized mammals", "fox, porcupine, possum, raccoon, skunk"), + ("non-insect invertebrates", "crab, lobster, snail, spider, worm"), + ("people", "baby, boy, girl, man, woman"), + ("reptiles", "crocodile, dinosaur, lizard, snake, turtle"), + ("small mammals", "hamster, mouse, rabbit, shrew, squirrel"), + ("trees", "maple, oak, palm, pine, willow"), + ("vehicles 1", "bicycle, bus, motorcycle, pickup truck, train"), + ("vehicles 2", "lawn-mower, rocket, streetcar, tank, tractor") +] + + +def create_fine_labels(): + res = [] + for l in LABELS: + for a in l[1].split(','): + res.append(a.strip()) + res = sorted(res) + # swap cups and crocodile + res[28], res[29] = res[29], res[28] + return res + + +def print_fine_labels(fine_labels): + i = 0 + for f in fine_labels: + print i, f + i += 1 + print + + +fine_labels = create_fine_labels() +print_fine_labels(fine_labels) + +train_dataset = dsets.CIFAR100(root=DATA_PATH, + train=True, + transform=transforms.ToTensor(), + download=False) +plt.show() +for i in range(len(train_dataset)): + # i = np.random.choice(range(len(train_dataset))) + image, label = train_dataset[i] + if True: + plt.imshow(image.numpy().transpose(1,2,0)) + print label, fine_labels[label] + pass diff --git a/my-tutorials/common/__init__.py b/my-tutorials/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/my-tutorials/common/__init__.pyc b/my-tutorials/common/__init__.pyc new file mode 100644 index 00000000..feb59a9a Binary files /dev/null and b/my-tutorials/common/__init__.pyc differ diff --git a/my-tutorials/common/pytorch_helpers.py b/my-tutorials/common/pytorch_helpers.py new file mode 100644 index 00000000..7dc7b6db --- /dev/null +++ b/my-tutorials/common/pytorch_helpers.py @@ -0,0 +1,30 @@ +import torch.nn as nn + +DATA_PATH = '../tutorials/data/' + + +class LinearRegression(nn.Module): + def __init__(self, in_size, out_size): + super(LinearRegression, self).__init__() + self.linear = nn.Linear(in_size, out_size) + + def forward(self, x): + return self.linear(x) + + def __call__(self, *args, **kwargs): + return super(LinearRegression, self).__call__(*args, **kwargs) + + +# Neural Network Model (1 hidden layer) +class Net1(nn.Module): + def __init__(self, in_size, hidden_size, out_size): + super(Net1, self).__init__() + self.fc1 = nn.Linear(in_size, hidden_size) + self.relu = nn.ReLU() + self.fc2 = nn.Linear(hidden_size, out_size) + + def forward(self, x): + out = self.fc1(x) + out = self.relu(out) + out = self.fc2(out) + return out \ No newline at end of file diff --git a/my-tutorials/common/pytorch_helpers.pyc b/my-tutorials/common/pytorch_helpers.pyc new file mode 100644 index 00000000..75d47b12 Binary files /dev/null and b/my-tutorials/common/pytorch_helpers.pyc differ diff --git a/my-tutorials/linear_regression.py b/my-tutorials/linear_regression.py new file mode 100644 index 00000000..5496194f --- /dev/null +++ b/my-tutorials/linear_regression.py @@ -0,0 +1,102 @@ +import torch +import torchvision +import torch.nn as nn +import numpy as np +import torch.utils.data as data +import torchvision.transforms as transforms +import torchvision.datasets as dsets +from torch.autograd import Variable +import matplotlib.pyplot as plt +import random +from common.pytorch_helpers import LinearRegression +# define a line with (0,0), (1,1) +def line45(x): + return x + + +def randomize_point(x, y, dx_max, dy_max): + dx = 2 * random.random() - 1 + dx *= dx_max + dy = 2 * random.random() - 1 + dy *= dy_max + return x + dx, y + dy + + +def generate_points(x_min, x_max, f, N): + assert N > 1, 'number of points should be > 1' + dx = (x_max - x_min) / float(N-1) + x = x_min + points = [] + while x <= x_max: + y = f(x) + points.append((x, y)) + x += dx + return points + + +def randomize_points(points, dx_max, dy_max): + res = [] + for x, y in points: + res.append(randomize_point(x, y, dx_max, dy_max)) + return res + + +def split_x_y(points): + x = [] + y = [] + for p in points: + x.append(p[0]) + y.append(p[1]) + return x, y + + +def list_to_numpy(l, t=np.float32): + return np.array([[x] for x in l], dtype=t) + + +points = generate_points(0, 1, line45, 100) +# print points +x_p, y_p = split_x_y(points) + +points_rand = randomize_points(points, .2, .2) +# print points_rand +x_pr, y_pr = split_x_y(points_rand) + + +############ LINEAR REGRESSION ################# +# let's create linear regression for the randomized points +x_train = list_to_numpy(x_p) +print("X shape" + str(x_train.shape)) # (100, 1) +y_train = list_to_numpy(y_pr) + +# Linear (1 -> 1) +model = LinearRegression(x_train.shape[1], y_train.shape[1]) + +# define loss and optimizer +criterion = nn.MSELoss() +# stochastic gradient descent +learning_rate = 0.1 +optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) + +# train +num_epochs = 300 +inputs = Variable(torch.from_numpy(x_train)) +targets = Variable(torch.from_numpy(y_train)) +for epoch in range(num_epochs): + # vars: epoch, outputs loss + optimizer.zero_grad() # why do we have to clear grads from last step? + outputs = model(inputs) # this call self.forward + loss = criterion(outputs, targets) + loss.backward() + optimizer.step() + + if (epoch+1) % 10 == 0: + print('Epoch [%d/%d], Loss: %.5f' % (epoch+1, num_epochs, loss.data[0])) + + +predicted = model(Variable(torch.from_numpy(x_train))).data.numpy() +plt.plot(x_train, y_train, 'ro', label='Original data') +plt.plot(x_train, predicted, label='Fitted line') +plt.plot(x_train, y_p, label='Expected line') +plt.legend() +plt.show() diff --git a/my-tutorials/mnist_feedforward.py b/my-tutorials/mnist_feedforward.py new file mode 100644 index 00000000..bcca3da9 --- /dev/null +++ b/my-tutorials/mnist_feedforward.py @@ -0,0 +1,86 @@ +import torch +import torch.nn as nn +import torchvision.datasets as dsets +import torchvision.transforms as transforms +from torch.autograd import Variable +from common.pytorch_helpers import DATA_PATH, Net1 + + +# Hyper Parameters +input_size = 784 +hidden_size = 500 +num_classes = 10 +num_epochs = 5 +batch_size = 100 +learning_rate = 0.001 +model_state_dict_filename = 'models/mnist_feedforward-params.pkl' + +# MNIST Dataset +train_dataset = dsets.MNIST(root=DATA_PATH, + train=True, + transform=transforms.ToTensor(), + download=True) + +test_dataset = dsets.MNIST(root=DATA_PATH, + train=False, + transform=transforms.ToTensor()) + +# Data Loader (Input Pipeline) +train_loader = torch.utils.data.DataLoader(dataset=train_dataset, + batch_size=batch_size, + shuffle=True) + +test_loader = torch.utils.data.DataLoader(dataset=test_dataset, + batch_size=batch_size, + shuffle=False) + + +model = Net1(input_size, hidden_size, num_classes) +if torch.cuda.is_available(): + model.cuda() + +# Loss and optimizer +criterion = nn.CrossEntropyLoss() +optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + +# Train +training = True +if training: + for epoch in range(num_epochs): + for i, (images, labels) in enumerate(train_loader): + images = Variable(images.view(-1, 28*28)) + if torch.cuda.is_available(): + images = images.cuda() + labels = Variable(labels) + if torch.cuda.is_available(): + labels = labels.cuda() + + # forward bacward optimize + optimizer.zero_grad() + outputs = model(images) + loss =criterion(outputs, labels) + loss.backward() + optimizer.step() + + if (i+1) % 100 == 0: + print ('Epoch [%d/%d], Step [%d/%d], Loss: %.4f' + %(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, loss.data[0])) + # Save the Model + torch.save(model.state_dict(), model_state_dict_filename) +else: + model.load_state_dict(torch.load(model_state_dict_filename)) + +# Test model +correct = total = 0 +for images, labels in test_loader: + images = Variable(images.view(-1, 28*28)) + if torch.cuda.is_available(): + images = images.cuda() + outputs = model(images) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted.cpu() == labels).sum() + +print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total)) + + diff --git a/my-tutorials/mnist_logistic.py b/my-tutorials/mnist_logistic.py new file mode 100644 index 00000000..17a1ef12 --- /dev/null +++ b/my-tutorials/mnist_logistic.py @@ -0,0 +1,80 @@ +""" +logistic regression (or logit) is a regression where output is categorical +""" +import torch +import torch.nn as nn +import torchvision.datasets as dsets +import torchvision.transforms as transforms +from torch.autograd import Variable +from common.pytorch_helpers import DATA_PATH, LinearRegression + +# Hyper Parameters +input_size = 784 # 28 * 28 +num_classes = 10 +num_epochs = 5 +batch_size = 100 + +# MNIST Dataset (Images and Labels) +train_dataset = dsets.MNIST(root=DATA_PATH, + train=True, + transform=transforms.ToTensor(), + download=True) + +test_dataset = dsets.MNIST(root=DATA_PATH, + train=False, + transform=transforms.ToTensor(), + download=False) + +# Dataset Loader (Input Pipline) +train_loader = torch.utils.data.DataLoader(dataset=train_dataset, + batch_size=batch_size, + shuffle=True) + +test_loader = torch.utils.data.DataLoader(dataset=test_dataset, + batch_size=batch_size, + shuffle=False) + +model = LinearRegression(input_size, num_classes) + +training = not True +model_state_dict_filename = "models/mnist_logistic-params.pkl" + +if training: + criterion = nn.CrossEntropyLoss() + learning_rate = 0.1 + + optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) + + # Train model + for epoch in range(num_epochs): + for i, (images, labels) in enumerate(train_loader): + images = Variable(images.view(-1, 28 * 28)) + labels = Variable(labels) + + # forward, backward, optimize + optimizer.zero_grad() + outputs = model(images) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + if (i+1) % 100 == 0: + print ('Epoch: [%d/%d], Step: [%d/%d], Loss: %.4f' + % (epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, loss.data[0])) + + # Save model + torch.save(model.state_dict(), model_state_dict_filename) +else: + model.load_state_dict(torch.load(model_state_dict_filename)) + +# Test model +correct = 0 +total = 0 +for images, labels in test_loader: + images = Variable(images.view(-1, 28 * 28)) # size [100, 784] + outputs = model(images) # size [100, 10] + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum() + +print('Accuracy of the model on the 10000 test images: %d %%' % (100 * correct / total)) \ No newline at end of file diff --git a/my-tutorials/models/README.txt b/my-tutorials/models/README.txt new file mode 100644 index 00000000..703b59a7 --- /dev/null +++ b/my-tutorials/models/README.txt @@ -0,0 +1,2 @@ +all pkl models will be placed here + diff --git a/tutorials/00 - PyTorch Basics/basics.ipynb b/tutorials/00_PyTorch_Basics/basics.ipynb similarity index 100% rename from tutorials/00 - PyTorch Basics/basics.ipynb rename to tutorials/00_PyTorch_Basics/basics.ipynb diff --git a/tutorials/00 - PyTorch Basics/main.py b/tutorials/00_PyTorch_Basics/main.py similarity index 87% rename from tutorials/00 - PyTorch Basics/main.py rename to tutorials/00_PyTorch_Basics/main.py index 17a50707..bf45845e 100644 --- a/tutorials/00 - PyTorch Basics/main.py +++ b/tutorials/00_PyTorch_Basics/main.py @@ -6,6 +6,7 @@ import torchvision.transforms as transforms import torchvision.datasets as dsets from torch.autograd import Variable +import matplotlib.pyplot as plt #========================== Table of Contents ==========================# @@ -20,20 +21,26 @@ #======================= Basic autograd example 1 =======================# # Create tensors. -x = Variable(torch.Tensor([1]), requires_grad=True) +x = Variable(torch.Tensor([3]), requires_grad=True) w = Variable(torch.Tensor([2]), requires_grad=True) b = Variable(torch.Tensor([3]), requires_grad=True) # Build a computational graph. y = w * x + b # y = 2 * x + 3 -# Compute gradients. +# Compute all gradients and store them in Variable.grad +# computes: x.grad = dy/dx, w.grad = dy/dw, b.grad = dy/db y.backward() # Print out the gradients. -print(x.grad) # x.grad = 2 -print(w.grad) # w.grad = 1 -print(b.grad) # b.grad = 1 +print("dy/dx = w") +print(x.grad) + +print("dy/dw = x") +print(w.grad) + +print("dy/db = 1") +print(b.grad) #======================== Basic autograd example 2 =======================# @@ -85,7 +92,23 @@ #===================== Implementing the input pipline =====================# # Download and construct dataset. -train_dataset = dsets.CIFAR10(root='../data/', +""" +https://www.cs.toronto.edu/~kriz/cifar.html +The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, +with 6000 images per class. +There are 50000 training images and 10000 test images. 10 classes: +0 - airplaine +1 - automobile +2 - bird +3 - cat +4 - deer +5 - dog +6 - frog +7 - horse +8 - ship +9 - truck +""" +train_dataset = dsets.CIFAR100(root='../data/', train=True, transform=transforms.ToTensor(), download=True) @@ -108,7 +131,7 @@ images, labels = data_iter.next() # Actual usage of data loader is as below. -for images, labels in train_loader: +for image, label in train_loader: # Your training code will be written here pass diff --git a/tutorials/01 - Linear Regression/main.py b/tutorials/01_Linear_Regression/main.py similarity index 100% rename from tutorials/01 - Linear Regression/main.py rename to tutorials/01_Linear_Regression/main.py diff --git a/tutorials/02 - Logistic Regression/main.py b/tutorials/02_Logistic_Regression/main.py similarity index 100% rename from tutorials/02 - Logistic Regression/main.py rename to tutorials/02_Logistic_Regression/main.py diff --git a/tutorials/03 - Feedforward Neural Network/main-gpu.py b/tutorials/03_Feedforward_Neural_Network/main-gpu.py similarity index 100% rename from tutorials/03 - Feedforward Neural Network/main-gpu.py rename to tutorials/03_Feedforward_Neural_Network/main-gpu.py diff --git a/tutorials/03 - Feedforward Neural Network/main.py b/tutorials/03_Feedforward_Neural_Network/main.py similarity index 100% rename from tutorials/03 - Feedforward Neural Network/main.py rename to tutorials/03_Feedforward_Neural_Network/main.py diff --git a/tutorials/04 - Convolutional Neural Network/main-gpu.py b/tutorials/04_Convolutional_Neural_Network/main-gpu.py similarity index 100% rename from tutorials/04 - Convolutional Neural Network/main-gpu.py rename to tutorials/04_Convolutional_Neural_Network/main-gpu.py diff --git a/tutorials/04 - Convolutional Neural Network/main.py b/tutorials/04_Convolutional_Neural_Network/main.py similarity index 100% rename from tutorials/04 - Convolutional Neural Network/main.py rename to tutorials/04_Convolutional_Neural_Network/main.py diff --git a/tutorials/05 - Deep Residual Network/main-gpu.py b/tutorials/05_Deep_Residual_Network/main-gpu.py similarity index 100% rename from tutorials/05 - Deep Residual Network/main-gpu.py rename to tutorials/05_Deep_Residual_Network/main-gpu.py diff --git a/tutorials/05 - Deep Residual Network/main.py b/tutorials/05_Deep_Residual_Network/main.py similarity index 100% rename from tutorials/05 - Deep Residual Network/main.py rename to tutorials/05_Deep_Residual_Network/main.py diff --git a/tutorials/06 - Recurrent Neural Network/main-gpu.py b/tutorials/06_Recurrent_Neural_Network/main-gpu.py similarity index 100% rename from tutorials/06 - Recurrent Neural Network/main-gpu.py rename to tutorials/06_Recurrent_Neural_Network/main-gpu.py diff --git a/tutorials/06 - Recurrent Neural Network/main.py b/tutorials/06_Recurrent_Neural_Network/main.py similarity index 100% rename from tutorials/06 - Recurrent Neural Network/main.py rename to tutorials/06_Recurrent_Neural_Network/main.py diff --git a/tutorials/07 - Bidirectional Recurrent Neural Network/main-gpu.py b/tutorials/07_Bidirectional_Recurrent_Neural_Network/main-gpu.py similarity index 100% rename from tutorials/07 - Bidirectional Recurrent Neural Network/main-gpu.py rename to tutorials/07_Bidirectional_Recurrent_Neural_Network/main-gpu.py diff --git a/tutorials/07 - Bidirectional Recurrent Neural Network/main.py b/tutorials/07_Bidirectional_Recurrent_Neural_Network/main.py similarity index 100% rename from tutorials/07 - Bidirectional Recurrent Neural Network/main.py rename to tutorials/07_Bidirectional_Recurrent_Neural_Network/main.py diff --git a/tutorials/08 - Language Model/data/train.txt b/tutorials/08_Language_Model/data/train.txt similarity index 100% rename from tutorials/08 - Language Model/data/train.txt rename to tutorials/08_Language_Model/data/train.txt diff --git a/tutorials/08 - Language Model/data_utils.py b/tutorials/08_Language_Model/data_utils.py similarity index 100% rename from tutorials/08 - Language Model/data_utils.py rename to tutorials/08_Language_Model/data_utils.py diff --git a/tutorials/08 - Language Model/main-gpu.py b/tutorials/08_Language_Model/main-gpu.py similarity index 100% rename from tutorials/08 - Language Model/main-gpu.py rename to tutorials/08_Language_Model/main-gpu.py diff --git a/tutorials/08 - Language Model/main.py b/tutorials/08_Language_Model/main.py similarity index 100% rename from tutorials/08 - Language Model/main.py rename to tutorials/08_Language_Model/main.py diff --git a/tutorials/09 - Image Captioning/README.md b/tutorials/09_Image_Captioning/README.md similarity index 100% rename from tutorials/09 - Image Captioning/README.md rename to tutorials/09_Image_Captioning/README.md diff --git a/tutorials/09 - Image Captioning/build_vocab.py b/tutorials/09_Image_Captioning/build_vocab.py similarity index 100% rename from tutorials/09 - Image Captioning/build_vocab.py rename to tutorials/09_Image_Captioning/build_vocab.py diff --git a/tutorials/09 - Image Captioning/data_loader.py b/tutorials/09_Image_Captioning/data_loader.py similarity index 100% rename from tutorials/09 - Image Captioning/data_loader.py rename to tutorials/09_Image_Captioning/data_loader.py diff --git a/tutorials/09 - Image Captioning/download.sh b/tutorials/09_Image_Captioning/download.sh similarity index 100% rename from tutorials/09 - Image Captioning/download.sh rename to tutorials/09_Image_Captioning/download.sh diff --git a/tutorials/09 - Image Captioning/model.py b/tutorials/09_Image_Captioning/model.py similarity index 100% rename from tutorials/09 - Image Captioning/model.py rename to tutorials/09_Image_Captioning/model.py diff --git a/tutorials/09 - Image Captioning/requirements.txt b/tutorials/09_Image_Captioning/requirements.txt similarity index 100% rename from tutorials/09 - Image Captioning/requirements.txt rename to tutorials/09_Image_Captioning/requirements.txt diff --git a/tutorials/09 - Image Captioning/resize.py b/tutorials/09_Image_Captioning/resize.py similarity index 100% rename from tutorials/09 - Image Captioning/resize.py rename to tutorials/09_Image_Captioning/resize.py diff --git a/tutorials/09 - Image Captioning/sample.py b/tutorials/09_Image_Captioning/sample.py similarity index 100% rename from tutorials/09 - Image Captioning/sample.py rename to tutorials/09_Image_Captioning/sample.py diff --git a/tutorials/09 - Image Captioning/train.py b/tutorials/09_Image_Captioning/train.py similarity index 100% rename from tutorials/09 - Image Captioning/train.py rename to tutorials/09_Image_Captioning/train.py diff --git a/tutorials/10 - Generative Adversarial Network/main-gpu.py b/tutorials/10_Generative_Adversarial_Network/main-gpu.py similarity index 100% rename from tutorials/10 - Generative Adversarial Network/main-gpu.py rename to tutorials/10_Generative_Adversarial_Network/main-gpu.py diff --git a/tutorials/10 - Generative Adversarial Network/main.py b/tutorials/10_Generative_Adversarial_Network/main.py similarity index 100% rename from tutorials/10 - Generative Adversarial Network/main.py rename to tutorials/10_Generative_Adversarial_Network/main.py diff --git a/tutorials/11 - Deep Convolutional Generative Adversarial Network/main-gpu.py b/tutorials/11_Deep_Convolutional_Generative_Adversarial_Network/main-gpu.py similarity index 100% rename from tutorials/11 - Deep Convolutional Generative Adversarial Network/main-gpu.py rename to tutorials/11_Deep_Convolutional_Generative_Adversarial_Network/main-gpu.py diff --git a/tutorials/11 - Deep Convolutional Generative Adversarial Network/main.py b/tutorials/11_Deep_Convolutional_Generative_Adversarial_Network/main.py similarity index 100% rename from tutorials/11 - Deep Convolutional Generative Adversarial Network/main.py rename to tutorials/11_Deep_Convolutional_Generative_Adversarial_Network/main.py diff --git a/tutorials/12 - Deep Q Network/dqn13.py b/tutorials/12_Deep_Q_Network/dqn13.py similarity index 100% rename from tutorials/12 - Deep Q Network/dqn13.py rename to tutorials/12_Deep_Q_Network/dqn13.py