Skip to content

Commit

Permalink
by minqi
Browse files Browse the repository at this point in the history
  • Loading branch information
@Minqi824
Minqi824 committed Aug 11, 2023
1 parent 05484a5 commit 7220cab
Show file tree
Hide file tree
Showing 5 changed files with 188 additions and 65 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ Each part of the pipeline can be instantiated by multiple components (core compo
||Learning Rate|[1e-2, 1e-3]|
||Weight Decay|[1e-2, 1e-4]|

## Quick Start with ADGym
## Quick Start with ADGym!

- For the experimental results of all the components, open the [test_ADGym.py](gym.py) and run:
```python
Expand Down
99 changes: 90 additions & 9 deletions components.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import os
import sys
from copy import deepcopy
import numpy as np
import pandas as pd
import random
import torch
from torch import nn
from networks import MLP, MLP_pair, AE
from networks import Pretrained_Model, Pretrained_Model_ResNet
import rtdl

from imblearn.over_sampling import SMOTE
Expand Down Expand Up @@ -119,7 +121,7 @@ def gym(self, mode='small'):
gyms['dropout'] = [0.0, 0.1, 0.2]
# gyms['network_initialization'] = ['default', 'xavier_uniform', 'xavier_normal',
# 'kaiming_uniform', 'kaiming_normal']
gyms['network_initialization'] = ['default', 'xavier_normal', 'kaiming_normal']
gyms['network_initialization'] = ['default', 'pretrained', 'xavier_normal', 'kaiming_normal']

## network training ##
gyms['training_strategy'] = [None]
Expand Down Expand Up @@ -400,21 +402,52 @@ def f_loss(self, s, y):

return loss

def f_optimizer(self):
def f_optimizer(self, pretrained=False):
if self.optimizer_name == 'SGD':
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, weight_decay=self.weight_decay)

if pretrained:
self.optimizer_pretrained = torch.optim.SGD(self.model_pretrained.parameters(), lr=self.lr, weight_decay=self.weight_decay)
else:
self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, weight_decay=self.weight_decay)
elif self.optimizer_name == 'Adam':
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr, weight_decay=self.weight_decay)

if pretrained:
self.optimizer_pretrained = torch.optim.Adam(self.model_pretrained.parameters(), lr=self.lr, weight_decay=self.weight_decay)
else:
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr, weight_decay=self.weight_decay)
elif self.optimizer_name == 'RMSprop':
self.optimizer = torch.optim.RMSprop(self.model.parameters(), lr=self.lr, weight_decay=self.weight_decay)

if pretrained:
self.optimizer_pretrained = torch.optim.RMSprop(self.model_pretrained.parameters(), lr=self.lr, weight_decay=self.weight_decay)
else:
self.optimizer = torch.optim.RMSprop(self.model.parameters(), lr=self.lr, weight_decay=self.weight_decay)
else:
raise NotImplementedError

return self

def f_pretrained(self):
criterion = nn.L1Loss()
for epoch in range(self.epochs):
loss_batch = []
for batch in self.train_loader:
# data
X, _ = [_.to(self.device) for _ in batch]
# clear gradient
self.model_pretrained.zero_grad()
# loss forward
if self.network_architecture == 'FTT':
X_hat = self.model_pretrained(x_num=X, x_cat=None).squeeze()
else:
X_hat = self.model_pretrained(X)

loss = criterion(X, X_hat)
# loss backward
loss.backward()
loss_batch.append(loss.item())
# gradient update
self.optimizer_pretrained.step()
print(f'Pretraining... Epoch: {epoch}, Training loss: {np.mean(loss_batch)}')

return self

def f_train(self):
self.utils.set_seed(self.seed)

Expand All @@ -426,7 +459,55 @@ def f_train(self):

# network initialization
self.f_network() # build network
self.model.apply(self.f_init_weights) # network weight initialization

if self.network_initialization == 'pretrained':
if self.model.__class__.__name__ == 'MLP':
decoder = nn.ModuleList()
for i in reversed(range(len(self.model.feature))):
l = deepcopy(self.model.feature[i])
dim_in, dim_out = l[0].out_features, l[0].in_features
l[0] = nn.Linear(dim_in, dim_out)
decoder.append(l)

self.model_pretrained = Pretrained_Model(encoder=self.model.feature, decoder=decoder)
self.f_optimizer(pretrained=True)
self.f_pretrained()
for l, params_pretrained in zip(self.model.feature, self.model_pretrained.encoder):
l.load_state_dict(params_pretrained.state_dict())

elif self.model.__class__.__name__ == 'AE':
self.model_pretrained = Pretrained_Model(encoder=self.model.encoder, decoder=self.model.decoder)
self.f_optimizer(pretrained=True)
self.f_pretrained()

for e, params_pretrained in zip(self.model.encoder, self.model_pretrained.encoder):
e.load_state_dict(params_pretrained.state_dict())
for d, params_pretrained in zip(self.model.decoder, self.model_pretrained.decoder):
d.load_state_dict(params_pretrained.state_dict())

elif self.model.__class__.__name__ == 'ResNet':
self.model_pretrained = Pretrained_Model_ResNet(input_size=self.data['X_train'].shape[1], model=self.model)
self.f_optimizer(pretrained=True)
self.f_pretrained()

self.model.first_layer.load_state_dict(self.model_pretrained.encoder_decoder[0].state_dict())
self.model.blocks.load_state_dict(self.model_pretrained.encoder_decoder[1].state_dict())

elif self.model.__class__.__name__ == 'FTTransformer':
self.model_pretrained = deepcopy(self.model)
self.model_pretrained.transformer.head = nn.Linear(8, self.data['X_train'].shape[1])
self.f_optimizer(pretrained=True)
self.f_pretrained()

self.model.feature_tokenizer.load_state_dict(self.model_pretrained.feature_tokenizer.state_dict())
self.model.cls_token.load_state_dict(self.model_pretrained.cls_token.state_dict())
self.model.transformer.blocks.load_state_dict(self.model_pretrained.transformer.blocks.state_dict())
else:
raise NotImplementedError

del self.model_pretrained; self.model.train()
else:
self.model.apply(self.f_init_weights) # network weight initialization

# optimizer
self.f_optimizer()
Expand Down
2 changes: 1 addition & 1 deletion gym.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,5 +245,5 @@ def run(self):

del data

adgym = ADGym(suffix='formal', la=5, grid_mode='large', grid_size=1000, seed_list=[1, 2, 3])
adgym = ADGym(suffix='test', la=10, grid_mode='large', grid_size=1000, seed_list=[1, 2, 3])
adgym.run()
107 changes: 53 additions & 54 deletions metaclassifier/meta_dl.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,75 +581,74 @@ def run(suffix, grid_mode, grid_size, mode, loss_name=None, ensemble=False, refi
# meta_baseline_rs_performance[i] = perf; del perf
# break

# rs: random search
meta_baseline_rs_performance[i] = np.nanmean(result_meta_baseline_test.loc[:, test_dataset])

# select the best components based on the performance in the training set of testing task (i.e., test dataset)
# ss: select the best components based on the performance in the training set of testing task (i.e., test dataset)
for _ in np.argsort(-result_meta_baseline_train.loc[:, test_dataset].values):
perf = result_meta_baseline_test.loc[_, test_dataset]
if not pd.isnull(perf):
meta_baseline_ss_performance[i] = perf; del perf
break

# ground truth
# gt: ground truth
perf = np.max(result_meta_baseline_test.loc[:, test_dataset])
meta_baseline_gt_performance[i] = perf; del perf

result_SOTA['Meta_baseline_rs'] = meta_baseline_rs_performance
result_SOTA['Meta_baseline_ss'] = meta_baseline_ss_performance
result_SOTA['Meta_baseline_gt'] = meta_baseline_gt_performance

# # run meta predictor
# run_meta = meta(seed=test_seed,
# metric=metric,
# suffix=suffix,
# grid_mode=grid_mode,
# grid_size=grid_size,
# loss_name=loss_name,
# ensemble=ensemble,
# refine=refine,
# test_dataset=test_dataset)
#
# try:
# if mode == 'two-stage':
# # retrain the meta predictor if we need to test on the new testing task
# if i == 0 or test_dataset != test_dataset_previous or test_seed != test_seed_previous:
# clf = run_meta.meta_fit()
# else:
# print('Using the trained meta predictor to predict...')
#
# clf.test_la = test_la
# perf = clf.meta_predict(metric=metric.lower())
#
# elif mode == 'end-to-end':
# # retrain the meta predictor if we need to test on the new testing task
# if i == 0 or test_dataset != test_dataset_previous or test_seed != test_seed_previous:
# clf = run_meta.meta_fit_end2end()
# else:
# print('Using the trained meta predictor to predict...')
#
# clf.test_la = test_la
# perf = clf.meta_predict_end2end(metric=metric.lower())
#
# else:
# raise NotImplementedError
#
# meta_classifier_performance[i] = perf
# except Exception as error:
# print(f'Something error when training meta-classifier: {error}')
# meta_classifier_performance[i] = -1
#
# result_SOTA['Meta'] = meta_classifier_performance
#
# if mode == 'two-stage':
# result_SOTA.to_csv('../result/' + file_path + '/' + metric + '-' + loss_name + '-' + str(ensemble)
# + '-' + str(refine) + '-meta-dl-twostage.csv', index=False)
# elif mode == 'end-to-end':
# result_SOTA.to_csv('../result/' + file_path + '/' + metric + '-' + loss_name + '-' + str(ensemble)
# + '-' + str(refine) + '-meta-dl-end2end.csv', index=False)
# else:
# raise NotImplementedError

result_SOTA.to_csv('../result/' + file_path + '/' + metric + '-debug.csv', index=False)
# run meta predictor
run_meta = meta(seed=test_seed,
metric=metric,
suffix=suffix,
grid_mode=grid_mode,
grid_size=grid_size,
loss_name=loss_name,
ensemble=ensemble,
refine=refine,
test_dataset=test_dataset)

try:
if mode == 'two-stage':
# retrain the meta predictor if we need to test on the new testing task
if i == 0 or test_dataset != test_dataset_previous or test_seed != test_seed_previous:
clf = run_meta.meta_fit()
else:
print('Using the trained meta predictor to predict...')

clf.test_la = test_la
perf = clf.meta_predict(metric=metric.lower())

elif mode == 'end-to-end':
# retrain the meta predictor if we need to test on the new testing task
if i == 0 or test_dataset != test_dataset_previous or test_seed != test_seed_previous:
clf = run_meta.meta_fit_end2end()
else:
print('Using the trained meta predictor to predict...')

clf.test_la = test_la
perf = clf.meta_predict_end2end(metric=metric.lower())

else:
raise NotImplementedError

meta_classifier_performance[i] = perf
except Exception as error:
print(f'Something error when training meta-classifier: {error}')
meta_classifier_performance[i] = -1

result_SOTA['Meta'] = meta_classifier_performance

if mode == 'two-stage':
result_SOTA.to_csv('../result/' + file_path + '/' + metric + '-' + loss_name + '-' + str(ensemble)
+ '-' + str(refine) + '-meta-dl-twostage.csv', index=False)
elif mode == 'end-to-end':
result_SOTA.to_csv('../result/' + file_path + '/' + metric + '-' + loss_name + '-' + str(ensemble)
+ '-' + str(refine) + '-meta-dl-end2end.csv', index=False)
else:
raise NotImplementedError

test_dataset_previous = test_dataset
test_seed_previous = test_seed
Expand Down
43 changes: 43 additions & 0 deletions networks.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,49 @@
import torch
from torch import nn

# Using Encoder-Decoder model structure for pretraining
class Pretrained_Model(nn.Module):
def __init__(self, encoder, decoder):
super(Pretrained_Model, self).__init__()
self.encoder = encoder
self.decoder = decoder

def forward(self, X):
# ModuleList
for e in self.encoder:
X = e(X)

for d in self.decoder:
X = d(X)
return X

class Pretrained_Model_ResNet(nn.Module):
def __init__(self, input_size, model):
super(Pretrained_Model_ResNet, self).__init__()
self.encoder_decoder = nn.Sequential(
model.first_layer,
model.blocks,
nn.Linear(128, input_size)
)

def forward(self, X):
X = self.encoder_decoder(X)
return X

class Pretrained_Model_FTT(nn.Module):
def __init__(self, input_size, model):
super(Pretrained_Model_FTT, self).__init__()
self.encoder_decoder = nn.Sequential(
model.feature_tokenizer,
model.cls_token,
model.transformer,
nn.Linear(8, input_size)
)

def forward(self, X):
X = self.encoder_decoder(X)
return X

# MLP backbone
class MLP(nn.Module):
def __init__(self, layers, input_size, hidden_size_list, act_fun, p):
Expand Down

0 comments on commit 7220cab

Please sign in to comment.