Commit 4339fe3d authored by Federico Betti's avatar Federico Betti
Browse files

first commit code

parent 06deb466
import torch
import numpy as np
import matplotlib.pyplot as plt
import random
import os
import argparse
parser = argparse.ArgumentParser(description='PyTorch Example',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--num-epochs', default=10, help='epoch number')
parser.add_argument('--batch-size', default=8, help='batch size')
parser.add_argument('--data-num', default=1000, help='number of data generated')
parser.add_argument('--lr', default=0.005, help='learning rate')
parser.add_argument('--plot-dir', default='', help='dir where to plot output')
parser.add_argument('--cuda', default=False, help='if to use cuda')
def set_all_seeds(seed):
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
if __name__ == '__main__':
args = parser.parse_args()
# PARAMETERS
batch_size = args.batch_size
epoch_num = args.num_epochs
data_num = args.data_num
learning_rate = args.lr
cuda = args.cuda
seed = 12345
set_all_seeds(seed)
# Create Data
X = np.random.rand(data_num, 1) * 2.0
w = np.random.rand(2, 1)
w_final, b_final = w[0], w[1]
y = X * w[0] + w[1] + np.random.randn(data_num, 1) * 0.05
print("Parametri Obiettivo", w[0], w[1])
# Creo parametri del modello
W = torch.rand(1, 1, requires_grad=True)
b = torch.rand(1, requires_grad=True)
# definisco il modello
def linear(x):
return torch.matmul(x, W) + b
# porto tutto sul device disponibile
if cuda and torch.cuda.is_available():
print("Tensors to cuda")
W = W.to('cuda')
b = b.to('cuda')
print(W)
else:
print("No cuda available")
# STORE DATA
loss_value_list = []
W_value_list = []
B_value_list = []
num_iterations = int(data_num / batch_size)
iteration = 0
# ciclo su ogni epoca
for epoch in range(epoch_num):
# ciclo per ogni batch
for batch_num in range(int(data_num / batch_size)):
# seleziono i dati corretti in base al batch
idx_start = batch_num * batch_size
idx_end = idx_start + batch_size
Xt = torch.from_numpy(X[idx_start: idx_end]).float()
yt = torch.from_numpy(y[idx_start: idx_end]).float()
# sposto i dati sul device corretto
if cuda and torch.cuda.is_available():
Xt = Xt.cuda()
yt = yt.cuda()
# Calcola le predizioni
y_pred = linear(Xt)
# Calcola funzione costo, Mean Squared Error
loss = torch.mean((y_pred - yt) ** 2)
if (iteration % 50) == 0:
print(loss.data, W.data, b.data)
# Esegui back-propagation
loss.backward()
# Aggiorna i parametri del modello
W.data = W.data - learning_rate * W.grad
b.data = b.data - learning_rate * b.grad
W_value_list.append(W.data.numpy()[0, 0])
B_value_list.append(b.data.numpy()[0])
loss_value_list.append(loss.data.item())
# Resetta il gradiente
W.grad.data.zero_()
b.grad.data.zero_()
iteration = iteration + 1
if args.plot_dir != "":
plt.plot(range(iteration), W_value_list)
plt.plot(range(iteration), [w_final] * iteration)
plt.savefig(args.plot_dir + "/W.png")
plt.clf()
plt.plot(range(iteration), B_value_list)
plt.plot(range(iteration), [b_final] * iteration)
plt.savefig(args.plot_dir + "/B.png")
plt.clf()
plt.plot(range(iteration), loss_value_list)
plt.savefig(args.plot_dir + "/loss.png")
plt.clf()
input_x = np.asarray(range(200)) / 100
plt.plot(X, y, '.')
plt.plot(input_x, input_x * W_value_list[-1] + B_value_list[-1], label='final prediction')
plt.plot(input_x, input_x * W_value_list[0] + B_value_list[0], label='initial prediction')
plt.plot(input_x, input_x * w_final + b_final, label='target')
plt.legend()
plt.savefig(args.plot_dir + "/data.png")
plt.clf()
import torch
import numpy as np
import matplotlib.pyplot as plt
import random
import os
import argparse
parser = argparse.ArgumentParser(description='PyTorch Example',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--num-epochs', default=10, help='epoch number')
parser.add_argument('--batch-size', default=8, help='batch size')
parser.add_argument('--data-num', default=1000, help='number of data generated')
parser.add_argument('--lr', default=0.005, help='learning rate')
parser.add_argument('--plot-dir', default='', help='dir where to plot output')
parser.add_argument('--cuda', default=False, help='if to use cuda')
def set_all_seeds(seed):
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
if __name__ == '__main__':
args = parser.parse_args()
# PARAMETERS
batch_size = args.batch_size
epoch_num = args.num_epochs
data_num = args.data_num
learning_rate = args.lr
cuda = args.cuda
seed = 12345
set_all_seeds(seed)
# Create Data
X = np.random.rand(data_num, 1) * 2.0
w = np.random.rand(2, 1)
w_final, b_final = w[0], w[1]
y = X * w[0] + w[1] + np.random.randn(data_num, 1) * 0.05
print("Parametri Obiettivo", w[0], w[1])
# il modello deve essere lo stesso della retta che ha generato i dati y = Wx + b
# Creo parametri del modello
...
# definisco il modello con funzioni pytorch
...
# porto tutto sul device disponibile
...
# STORE DATA
loss_value_list = []
W_value_list = []
B_value_list = []
num_iterations = int(data_num / batch_size)
iteration = 0
# ciclo su ogni epoca
...
# ciclo per ogni batch
...
# seleziono i dati corretti in base al batch
...
# sposto i dati sul device corretto
...
# Calcola le predizioni
...
# Calcola funzione costo, Mean Squared Error
...
if (iteration % 50) == 0:
print(loss.data, W.data, b.data)
# Esegui back-propagation
...
# Aggiorna i parametri del modello con standard SGD
...
W_value_list.append(W.data.numpy()[0, 0])
B_value_list.append(b.data.numpy()[0])
loss_value_list.append(loss.data.item())
# Resetta il gradiente
W.grad.data.zero_()
b.grad.data.zero_()
iteration = iteration + 1
if args.plot_dir != "":
plt.plot(range(iteration), W_value_list)
plt.plot(range(iteration), [w_final] * iteration)
plt.savefig(args.plot_dir + "/W.png")
plt.clf()
plt.plot(range(iteration), B_value_list)
plt.plot(range(iteration), [b_final] * iteration)
plt.savefig(args.plot_dir + "/B.png")
plt.clf()
plt.plot(range(iteration), loss_value_list)
plt.savefig(args.plot_dir + "/loss.png")
plt.clf()
input_x = np.asarray(range(200)) / 100
plt.plot(X, y, '.')
plt.plot(input_x, input_x * W_value_list[-1] + B_value_list[-1], label='final prediction')
plt.plot(input_x, input_x * W_value_list[0] + B_value_list[0], label='initial prediction')
plt.plot(input_x, input_x * w_final + b_final, label='target')
plt.legend()
plt.savefig(args.plot_dir + "/data.png")
plt.clf()
import math
import os
import random
import time
import torch
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data.distributed
from torch import nn
from torch.utils.tensorboard import SummaryWriter
from torchvision import datasets, transforms, models
from tqdm import tqdm
import argparse
import numpy as np
from util.metric_saving import Metric
parser = argparse.ArgumentParser(description='ImageNet Example',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--num-epochs', default=10, help='epoch number')
parser.add_argument('--batch-size', default=64, help='batch size')
parser.add_argument('--data-num', default=1000, help='number of data generated')
parser.add_argument('--lr', default=0.001, help='learning rate')
parser.add_argument('--plot-dir', default='', help='dir where to plot output')
parser.add_argument('--cuda', default=False, help='if to use cuda')
parser.add_argument('--train-dir', default="/m100_work/EUCC_staff/fbetti/pytorch/tiny-imagenet-200/train",
help='path to training data')
parser.add_argument('--val-dir', default="/m100_work/EUCC_staff/fbetti/pytorch/tiny-imagenet-200/val",
help='path to validation data')
parser.add_argument('--warmup-epochs', type=float, default=5, help='number of warmup epochs')
parser.add_argument('--log-dir', default='./logs', help='tensorboard log directory')
parser.add_argument('--run-name', default='test_random', help='tensorboard log directory name')
def set_all_seeds(seed):
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
for param in model.parameters():
param.requires_grad = False
def train(epoch):
# set the model in training mode
model.train()
train_loss = Metric('train_loss')
train_accuracy = Metric('train_accuracy')
iteration = epoch * len(train_loader)
with tqdm(total=len(train_loader), desc='Train Epoch #{}'.format(epoch + 1)) as t:
# iterate over train loader
for batch_idx, (data, target) in enumerate(train_loader):
# adjust learning rate manually
adjust_learning_rate(epoch, batch_idx)
if cuda:
data, target = data.cuda(), target.cuda()
# model prediction
output = model(data)
# calculate accuracy and update metric
accuracy_tensor = accuracy(output, target)
train_accuracy.update(accuracy_tensor)
# calculate loss and update metric. CrossEntropy loss needed.
loss = F.cross_entropy(output, target)
train_loss.update(loss)
# Calculate gradients
loss.backward()
# Apply gradient trough the optimizer
optimizer.step()
optimizer.zero_grad()
t.set_postfix({'loss': train_loss.avg.item(),
'accuracy': 100. * train_accuracy.avg.item()})
t.update(1)
if log_writer:
log_writer.add_scalar('train/loss', train_loss.last.item(), iteration)
log_writer.add_scalar('train/accuracy', 100. * train_accuracy.last.item(), iteration)
iteration += 1
def validate(epoch):
# set the model in evaluation mode
model.eval()
val_loss = Metric('val_loss')
val_accuracy = Metric('val_accuracy')
with tqdm(total=len(val_loader), desc='Validate Epoch #{}'.format(epoch + 1)) as t:
# do not compute grad
with torch.no_grad():
for data, target in val_loader:
if cuda:
data, target = data.cuda(), target.cuda()
output = model(data)
val_loss.update(F.cross_entropy(output, target))
val_accuracy.update(accuracy(output, target))
t.set_postfix({'loss': val_loss.avg.item(),
'accuracy': 100. * val_accuracy.avg.item()})
t.update(1)
if log_writer:
log_writer.add_scalar('val/loss', val_loss.avg, epoch)
log_writer.add_scalar('val/accuracy', val_accuracy.avg, epoch)
def adjust_learning_rate(epoch, batch_idx):
if epoch < warmup_epochs:
epoch += float(batch_idx + 1) / len(train_loader)
lr_adj = (epoch / warmup_epochs + 1)
elif epoch < 30:
lr_adj = 1.
elif epoch < 60:
lr_adj = 1e-1
elif epoch < 80:
lr_adj = 1e-2
else:
lr_adj = 1e-3
for param_group in optimizer.param_groups:
param_group['lr'] = base_lr * lr_adj
class Net(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(3, 64, 5)
self.batch1 = nn.BatchNorm2d(64)
self.conv2 = nn.Conv2d(64, 64, 3)
self.batch2 = nn.BatchNorm2d(64)
self.conv3 = nn.Conv2d(64, 128, 3)
self.batch3 = nn.BatchNorm2d(128)
self.drop1 = nn.Dropout(0.25)
self.conv2_1 = nn.Conv2d(128, 128, 3)
self.batch2_1 = nn.BatchNorm2d(128)
self.conv2_2 = nn.Conv2d(128, 256, 5, stride=2)
self.batch2_2 = nn.BatchNorm2d(256)
self.conv2_3 = nn.Conv2d(256, 256, 5)
self.batch2_3 = nn.BatchNorm2d(256)
self.drop2_1 = nn.Dropout(0.25)
self.conv3_1 = nn.Conv2d(256, 512, 3)
self.batch3_1 = nn.BatchNorm2d(512)
self.conv3_2 = nn.Conv2d(512, 512, 3)
self.batch3_2 = nn.BatchNorm2d(512)
self.conv3_3 = nn.Conv2d(512, 512, 3)
self.batch3_3 = nn.BatchNorm2d(512)
self.drop3_1 = nn.Dropout(0.25)
self.fc1 = nn.Linear(512 * 15 * 15, 512)
# self.batch4 = nn.BatchNorm1d(512)
self.drop4_1 = nn.Dropout(0.5)
self.fc2 = nn.Linear(512, 200)
def forward(self, x):
x = F.relu(self.batch1(self.conv1(x)))
x = F.relu(self.batch2(self.conv2(x)))
x = F.relu(self.batch3(self.conv3(x)))
x = self.drop1(x)
x = F.relu(self.batch2_1(self.conv2_1(x)))
x = F.relu(self.batch2_2(self.conv2_2(x)))
x = F.relu(self.batch2_3(self.conv2_3(x)))
x = self.drop2_1(x)
x = F.relu(self.batch3_1(self.conv3_1(x)))
x = F.relu(self.batch3_2(self.conv3_2(x)))
x = F.relu(self.batch3_3(self.conv3_3(x)))
x = self.drop3_1(x)
x = torch.flatten(x, 1) # flatten all dimensions except batch
x = F.relu(self.fc1(x))
x = self.drop4_1(x)
x = self.fc2(x)
return x
def get_model(resnet=False):
if resnet:
num_classes = 200
model = models.resnet50(True)
set_parameter_requires_grad(model, feature_extract)
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, num_classes)
else:
model = Net()
return model
# return models.resnet18()
def accuracy(output, target):
# get the index of the max log-probability
pred = output.max(1, keepdim=True)[1]
return pred.eq(target.view_as(pred)).cpu().float().mean()
if __name__ == '__main__':
args = parser.parse_args()
cuda = torch.cuda.is_available()
print("Cuda is available", cuda)
log_dir = args.log_dir
batch_size = args.batch_size
base_lr = args.lr
seed = 12345
train_dir = args.train_dir # "/m100_work/EUCC_staff/fbetti/pytorch/tiny-imagenet-200/train"
val_dir = args.val_dir # "/m100_work/EUCC_staff/fbetti/pytorch/tiny-imagenet-200/val"
epochs = args.num_epochs
warmup_epochs = args.warmup_epochs
if args.run_name == "":
run_name = time.time()
else:
run_name = args.run_name
summary_writer_dir = os.path.join(log_dir, run_name)
os.makedirs(summary_writer_dir, exist_ok=True)
set_all_seeds(seed)
cudnn.benchmark = True
# Define Tensorboard writer
log_writer = SummaryWriter(summary_writer_dir)
crop_size = 64
test_image_size = 64
# load train dataset, make transformation and construct data_loader
train_dataset = \
datasets.ImageFolder(train_dir,
transform=transforms.Compose([
transforms.RandomResizedCrop(crop_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
# load validation dataset, make transformation and construct data_loader
val_dataset = \
datasets.ImageFolder(val_dir,
transform=transforms.Compose([
transforms.Resize(test_image_size),
transforms.CenterCrop(crop_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True)
feature_extract = True
# define model
model = get_model(True)
print(model)
params_to_update = model.parameters()
print("Params to learn:")
if feature_extract:
params_to_update = []
for name, param in model.named_parameters():
if param.requires_grad == True:
params_to_update.append(param)
print("\t", name)
else:
for name, param in model.named_parameters():
if param.requires_grad == True:
print("\t", name)
# Move model to GPU.
if cuda:
model.cuda()
# define optimizer
optimizer = optim.SGD(params_to_update, lr=base_lr, momentum=0)
# optimizer = optim.Adam(model.parameters())
# optimizer = optim.Adagrad(model.parameters())
# training loop
for epoch in range(epochs):
train(epoch)
validate(epoch)
import math
import torch
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data.distributed
from torch.utils.tensorboard import SummaryWriter
from torchvision import datasets, transforms, models
from tqdm import tqdm
import argparse
from util.metric_saving import Metric
parser = argparse.ArgumentParser(description='ImageNet Example',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--num-epochs', default=10, help='epoch number')
parser.add_argument('--batch-size', default=64, help='batch size')
parser.add_argument('--data-num', default=1000, help='number of data generated')
parser.add_argument('--lr', default=0.005, help='learning rate')
parser.add_argument('--plot-dir', default='', help='dir where to plot output')
parser.add_argument('--cuda', default=False, help='if to use cuda')
parser.add_argument('--train-dir', default="/m100_work/EUCC_staff/fbetti/pytorch/tiny-imagenet-200/train",
help='path to training data')
parser.add_argument('--val-dir', default="/m100_work/EUCC_staff/fbetti/pytorch/tiny-imagenet-200/val",
help='path to validation data')
parser.add_argument('--warmup-epochs', type=float, default=5, help='number of warmup epochs')
parser.add_argument('--log-dir', default='./logs', help='tensorboard log directory')
def train(epoch):
# set the model in training mode
...