diff --git a/batch_iterators.py b/batch_iterators.py index cd16bc7..8848ce9 100644 --- a/batch_iterators.py +++ b/batch_iterators.py @@ -177,7 +177,6 @@ def __call__(self, X, y=None): self.y = y else: self.y = y[0] - print X self.X = X return self diff --git a/convnets/models.py b/convnets/models.py index 2a74594..76f2d34 100644 --- a/convnets/models.py +++ b/convnets/models.py @@ -2057,6 +2057,278 @@ def net50(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_le layer = GlobalPoolLayer(layer,pool_function=Tmean4) layer = NonlinearityLayer(layer,nonlinearity=nonlinearities.softmax) + net = NeuralNet( + layer, + update=adam, + update_learning_rate=0.001, + #update_momentum=0.9, + regression=False, + max_epochs=100, + verbose=1, + on_epoch_finished=[EarlyStopping(patience=10),], + batch_iterator_train = batch_iterator_train, + batch_iterator_test = batch_iterator_test, + train_split = train_split, + ) + return net + +def net51(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_leaky_rectify, + train_split=TrainSplit(0.2), + batch_iterator_train=BatchIterator(batch_size=256),batch_iterator_test=BatchIterator(batch_size=256)): + # net13 with henorm + layer = InputLayer(shape=(None, n_channels, width, height)) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(64,1), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,64), pad=1, num_filters=32) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=None, W=he_norm, filter_size=(3,3), pad=1, num_filters=n_output) + layer = GlobalPoolLayer(layer) + layer = NonlinearityLayer(layer,nonlinearity=nonlinearities.softmax) + + net = NeuralNet( + layer, + update=adam, + update_learning_rate=0.001, + #update_momentum=0.9, + regression=False, + max_epochs=100, + verbose=1, + on_epoch_finished=[EarlyStopping(patience=10),], + batch_iterator_train = batch_iterator_train, + batch_iterator_test = batch_iterator_test, + train_split = train_split, + ) + return net + +def net52(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_leaky_rectify, + train_split=TrainSplit(0.2), + batch_iterator_train=BatchIterator(batch_size=256),batch_iterator_test=BatchIterator(batch_size=256)): + # net13 with henorm + layer = InputLayer(shape=(None, n_channels, width, height)) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(64,1), pad=1, num_filters=4) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,64), pad=1, num_filters=8) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=16) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=None, W=he_norm, filter_size=(3,3), pad=1, num_filters=n_output) + layer = GlobalPoolLayer(layer) + layer = NonlinearityLayer(layer,nonlinearity=nonlinearities.softmax) + + net = NeuralNet( + layer, + update=adam, + update_learning_rate=0.001, + #update_momentum=0.9, + regression=False, + max_epochs=100, + verbose=1, + on_epoch_finished=[EarlyStopping(patience=10),], + batch_iterator_train = batch_iterator_train, + batch_iterator_test = batch_iterator_test, + train_split = train_split, + ) + return net + + +def net53(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_leaky_rectify, + train_split=TrainSplit(0.2), + batch_iterator_train=BatchIterator(batch_size=256),batch_iterator_test=BatchIterator(batch_size=256)): + #from net41 with more filters in 1x1 layers + layer = InputLayer(shape=(None, n_channels, width, height)) + # channel reduction == feature map reduction + layer = Conv2DLayer(layer, nonlinearity=None, filter_size=(1,1), num_filters=8) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=None, W=he_norm, filter_size=(3,3), pad=1, num_filters=n_output) + layer = GlobalPoolLayer(layer) + layer = NonlinearityLayer(layer,nonlinearity=nonlinearities.softmax) + + net = NeuralNet( + layer, + update=adam, + update_learning_rate=0.001, + #update_momentum=0.9, + regression=False, + max_epochs=100, + verbose=1, + on_epoch_finished=[EarlyStopping(patience=10),], + batch_iterator_train = batch_iterator_train, + batch_iterator_test = batch_iterator_test, + train_split = train_split, + ) + return net + +def net54(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_leaky_rectify, + train_split=TrainSplit(0.2), + batch_iterator_train=BatchIterator(batch_size=256),batch_iterator_test=BatchIterator(batch_size=256)): + #from net29 seperate input layers for each channel + layer = InputLayer(shape=(None, n_channels, width, height)) + # channel reduction == feature map reduction + layer = Conv2DLayer(layer, nonlinearity=None, filter_size=(1,1), num_filters=4) + # reduction in freq components + layer = Conv2DLayer(layer, nonlinearity=None, filter_size=(1,64), stride=(1,1), num_filters=16) + #reshape to a 2D image with only one channel + layer = ReshapeLayer(layer, ([0], 1, [2], -1)) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, stride=(2,2), filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=None, W=he_norm, filter_size=(3,3), pad=1, num_filters=n_output) + layer = GlobalPoolLayer(layer) + layer = NonlinearityLayer(layer,nonlinearity=nonlinearities.softmax) + + net = NeuralNet( + layer, + update=adam, + update_learning_rate=0.001, + #update_momentum=0.9, + regression=False, + max_epochs=100, + verbose=1, + on_epoch_finished=[EarlyStopping(patience=10),], + batch_iterator_train = batch_iterator_train, + batch_iterator_test = batch_iterator_test, + train_split = train_split, + ) + return net + +def net55(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_leaky_rectify, + train_split=TrainSplit(0.2), + batch_iterator_train=BatchIterator(batch_size=256),batch_iterator_test=BatchIterator(batch_size=256)): + #from net29 seperate input layers for each channel + layer = InputLayer(shape=(None, n_channels, width, height)) + # channel reduction == feature map reduction + layer = Conv2DLayer(layer, nonlinearity=None, filter_size=(1,1), num_filters=4) + # reduction in freq components + layer = Conv2DLayer(layer, nonlinearity=None, filter_size=(1,64), stride=(1,1), num_filters=16) + #reshape to a 2D image with only one channel + layer = ReshapeLayer(layer, ([0], 1, [2], -1)) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = MaxPool2DLayer(layer, pool_size=(2,2)) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = MaxPool2DLayer(layer, pool_size=(2,2)) + layer = dropout(layer,p=0.5) + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(3,3), pad=1, num_filters=32) + layer = Conv2DLayer(layer, nonlinearity=None, W=he_norm, filter_size=(3,3), pad=1, num_filters=n_output) + layer = GlobalPoolLayer(layer) + layer = NonlinearityLayer(layer,nonlinearity=nonlinearities.softmax) + + net = NeuralNet( + layer, + update=adam, + update_learning_rate=0.001, + #update_momentum=0.9, + regression=False, + max_epochs=100, + verbose=1, + on_epoch_finished=[EarlyStopping(patience=10),], + batch_iterator_train = batch_iterator_train, + batch_iterator_test = batch_iterator_test, + train_split = train_split, + ) + return net + +def net56(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_leaky_rectify, + train_split=TrainSplit(0.2), + batch_iterator_train=BatchIterator(batch_size=256),batch_iterator_test=BatchIterator(batch_size=256)): + #from net29 seperate input layers for each channel + layer = InputLayer(shape=(None, n_channels, width, height)) + # channel reduction == feature map reduction + layer = Conv2DLayer(layer, nonlinearity=None, filter_size=(1,1), num_filters=4) + + # reduction in freq components #reshape to a 2D image with only one channel + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,height), stride=(1,1), num_filters=32) + layer = ReshapeLayer(layer, ([0], 1, [2], -1)) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(width,1), stride=(1,1), num_filters=32) + layer = ReshapeLayer(layer, ([0], 1, -1, [3])) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,32), stride=(1,1), num_filters=16) + layer = ReshapeLayer(layer, ([0], 1, [2], -1)) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(32,1), stride=(1,1), num_filters=16) + layer = ReshapeLayer(layer, ([0], 1, -1, [3])) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,16), stride=(1,1), num_filters=8) + layer = ReshapeLayer(layer, ([0], 1, [2], -1)) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(16,1), stride=(1,1), num_filters=8) + layer = ReshapeLayer(layer, ([0], 1, -1, [3])) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(8,8), pad=1, num_filters=16) + layer = Conv2DLayer(layer, nonlinearity=None, W=he_norm, filter_size=(1,1), pad=1, num_filters=n_output) + layer = GlobalPoolLayer(layer) + layer = NonlinearityLayer(layer,nonlinearity=nonlinearities.softmax) + + net = NeuralNet( + layer, + update=adam, + update_learning_rate=0.001, + #update_momentum=0.9, + regression=False, + max_epochs=100, + verbose=1, + on_epoch_finished=[EarlyStopping(patience=10),], + batch_iterator_train = batch_iterator_train, + batch_iterator_test = batch_iterator_test, + train_split = train_split, + ) + return net + +def net57(n_channels,width,height,n_output=2,nonlinearity=nonlinearities.very_leaky_rectify, + train_split=TrainSplit(0.2), + batch_iterator_train=BatchIterator(batch_size=256),batch_iterator_test=BatchIterator(batch_size=256)): + #from net29 seperate input layers for each channel + layer = InputLayer(shape=(None, n_channels, width, height)) + # channel reduction == feature map reduction + layer = Conv2DLayer(layer, nonlinearity=None, filter_size=(1,1), num_filters=4) + + # reduction in freq components #reshape to a 2D image with only one channel + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,height), stride=(1,1), num_filters=32) + layer = ReshapeLayer(layer, ([0], 1, [2], -1)) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,32), stride=(1,1), num_filters=32) + layer = ReshapeLayer(layer, ([0], 1, [2], -1)) + layer = dropout(layer,p=0.5) + + layer = Conv2DLayer(layer, nonlinearity=nonlinearity, W=he_norm, filter_size=(1,32), stride=(1,1), num_filters=32) + layer = dropout(layer,p=0.5) + + gp_mean = GlobalPoolLayer(layer, pool_function=T.mean, name="gp_mean") + gp_max = GlobalPoolLayer(layer, pool_function=T.max, name="gp_max") + gp_min = GlobalPoolLayer(layer, pool_function=T.min, name="gp_min") + gp_var = GlobalPoolLayer(layer, pool_function=T.var, name="gp_var") + gp = ConcatLayer([gp_mean, gp_max, gp_min, gp_var], name="gp_concat") + + + layer = DenseLayer(gp, nonlinearity=nonlinearities.softmax, num_units=n_output, name="dense_softmax") + net = NeuralNet( layer, update=adam, diff --git a/train.py b/train.py index 8bd5587..9d35663 100644 --- a/train.py +++ b/train.py @@ -13,6 +13,7 @@ from tools.utils import calcFFT, rolling_window_ext from tools.analyze import read_data_1h, read_data +from train_split import TrainSplit from fractions import gcd import numpy as np @@ -27,6 +28,7 @@ #from memory_profiler import profile import psutil import scipy +import operator from sklearn.base import clone from datetime import datetime @@ -292,7 +294,6 @@ def set_white_lists(): print 'Warning: CSV file should only contain safe files.' print words - def allocate_array(): global extra_counter @@ -320,7 +321,6 @@ def read_extra_data(): print dataset.user, 'in', chs.keys() read_extra_dataset(dataset) - def read_test_data(dataset,start,stop): global ms_test global test_counter @@ -476,8 +476,6 @@ def preprocess(): } hkl.dump(data, 'preprocessedData.hkl',compression="lzf") - - def apply_normalization(data_in): global maximum global minimum @@ -657,14 +655,13 @@ def check(m_array): if np.sum(m)<0.1: print 'Warning: found a zero or near zero sum array' - -def test(): +def test(netSpec, fold, no_folds): print("Validating...") print "Changing batch iterator test:" from nolearn.lasagne import BatchIterator netSpec.batch_iterator_test = BatchIterator(batch_size=128) - train_split = TrainSplit(5,0) + train_split = TrainSplit(no_folds,fold) x_train, x_valid, dummy_0, dummy_1 = train_split([x],None) print "Calculating final prediction for the hour long sessions" @@ -735,29 +732,15 @@ def test(): from sklearn.metrics import roc_auc_score,log_loss print probabilities[:,1].shape print yVal.shape - print "roc_auc:", roc_auc_score(yVal, probabilities[:,1]) + roc_auc = roc_auc_score(yVal, probabilities[:,1]) + print "roc_auc:", roc_auc print "log_loss:", log_loss(yVal, probabilities[:,1]) - print len(probabilities_normal_hour) - print len(probabilities_seizure_hour) - print len(probabilities_xtra_seizure_hour) - - print 'probabilities_normal_hour' - for p in probabilities_normal_hour: - print p - - print 'probabilities_seizure_hour' - for p in probabilities_seizure_hour: - print p - - print 'probabilities_xtra_seizure_hour' - for p in probabilities_xtra_seizure_hour: - print p - yVal_hour = np.hstack((np.zeros(len(probabilities_normal_hour)),np.ones(len(probabilities_seizure_hour)),np.ones(len(probabilities_xtra_seizure_hour)))) probabilities_hour = probabilities_normal_hour + probabilities_seizure_hour + probabilities_xtra_seizure_hour - print "roc_auc for the hours:", roc_auc_score(yVal_hour, probabilities_hour) + roc_auc_hours = roc_auc_score(yVal_hour, probabilities_hour) + print "roc_auc for the hours:", roc_auc_hours print "log_loss for the hours:", log_loss(yVal_hour, probabilities_hour) print "saving predictions to csv file" @@ -798,14 +781,12 @@ def test(): csv.write(filename+','+str(probabilities_test[counter+i])+'\n') csv.close - + return roc_auc, roc_auc_hours data_path = args.data_path files_per_hour = 6 -#is_train_index = get_train_val_split(size) - if args.no_preprocessing: load_preprocessed() @@ -814,41 +795,57 @@ def test(): check_magnitudes() -model_training = None -model_evaluation = None +def train_and_test(fold=0,no_folds=5): + model_training = None + model_evaluation = None + + print "Building models ..." + if include_userdata: + import convnets.multi_user_models as cnmu + model_training = getattr(cnmu, cfg['training']['model']) + print "Model name for the training phase: ", cfg['training']['model'] + model_evaluation = getattr(cnmu, cfg['evaluation']['model']) + print "Model name for the evaluation phase: ", cfg['evaluation']['model'] + else: + import convnets.models as cn + model_training = getattr(cn, cfg['training']['model']) + print "Model name for the training phase: ", cfg['training']['model'] + model_evaluation = getattr(cn, cfg['evaluation']['model']) + print "Model name for the evaluation phase: ", cfg['evaluation']['model'] + + if args.mode=="single-channel": + no_channels = 1 + else: + no_channels = args.no_channels -print "Building models ..." -if include_userdata: - import convnets.multi_user_models as cnmu - model_training = getattr(cnmu, cfg['training']['model']) - print "Model name for the training phase: ", cfg['training']['model'] - model_evaluation = getattr(cnmu, cfg['evaluation']['model']) - print "Model name for the evaluation phase: ", cfg['evaluation']['model'] -else: - import convnets.models as cn - model_training = getattr(cn, cfg['training']['model']) - print "Model name for the training phase: ", cfg['training']['model'] - model_evaluation = getattr(cn, cfg['evaluation']['model']) - print "Model name for the evaluation phase: ", cfg['evaluation']['model'] - -if args.mode=="single-channel": - no_channels = 1 -else: - no_channels = args.no_channels + from batch_iterators import BI_new -from batch_iterators import BI_new -from train_split import TrainSplit + if args.no_training: + netSpec = model_evaluation(no_channels,m_window,ceil-floor, + train_split = TrainSplit(no_folds,fold), + batch_iterator_train=BI_new(16),batch_iterator_test=BI_new(128)) + netSpec, xTrain, xVal = load_trained_and_normalize(netSpec, xTrain, xVal) + else: + netSpec = model_training(no_channels,m_window,ceil-floor, + train_split = TrainSplit(no_folds,fold), + batch_iterator_train=BI_new(16),batch_iterator_test=BI_new(128)) + netSpec = train(netSpec) + + if args.chosen_validation_ratio != 0: + return test(netSpec, fold, no_folds) + +def geometric_mean(iterable): + return (reduce(operator.mul, iterable)) ** (1.0/len(iterable)) + +no_folds = 5 +roc_auc_lst = [] +roc_auc_hours_lst = [] +for i in range(no_folds): + roc_auc, roc_auc_hours = train_and_test(i,no_folds) + roc_auc_lst.append(roc_auc) + roc_auc_hours_lst.append(roc_auc_hours) + +print 'geomean_roc_auc: ', geometric_mean(roc_auc_lst) +print 'geomean_roc_auc_hours: ', geometric_mean(roc_auc_hours_lst) -if args.no_training: - netSpec = model_evaluation(no_channels,m_window,ceil-floor, - train_split = TrainSplit(5,0), - batch_iterator_train=BI_new(16),batch_iterator_test=BI_new(128)) - netSpec, xTrain, xVal = load_trained_and_normalize(netSpec, xTrain, xVal) -else: - netSpec = model_training(no_channels,m_window,ceil-floor, - train_split = TrainSplit(5,0), - batch_iterator_train=BI_new(16),batch_iterator_test=BI_new(128)) - netSpec = train(netSpec) -if args.chosen_validation_ratio != 0: - test()