make the configs run on stage2 data

EliasVansteenkiste · Apr 10, 2017 · d4913b4 · d4913b4
1 parent 2e663fd
commit d4913b4
Show file tree

Hide file tree

Showing 19 changed files with 54 additions and 47 deletions.
diff --git a/configs_class_dsb/dsb_a_eliasq10_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq10_mal2_s5_p8a1_all.py
@@ -76,12 +76,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
     return selected_candidates
 
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq10_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq10_mal2_s5_p8a1_spl.py
@@ -78,10 +78,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq11_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq11_mal2_s5_p8a1_all.py
@@ -69,12 +69,12 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
+
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq11_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq11_mal2_s5_p8a1_spl.py
@@ -69,10 +69,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq14_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq14_mal2_s5_p8a1_all.py
@@ -79,12 +79,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 batch_size = 1
 
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq14_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq14_mal2_s5_p8a1_spl.py
@@ -78,10 +78,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq15_mal7_s5_p8a1.py b/configs_class_dsb/dsb_a_eliasq15_mal7_s5_p8a1.py
@@ -261,7 +261,7 @@ def load_pretrained_model(l_in):
                 b=nn.init.Constant(0))
 
 
-    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/eavsteen/","r_fred_malignancy_7-20170404-163552.pkl"))
+    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/","r_fred_malignancy_7-20170404-163552.pkl"))
     nn.layers.set_all_param_values(l, metadata['param_values'])
 
     return l

diff --git a/configs_class_dsb/dsb_a_eliasq15_mal7_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq15_mal7_s5_p8a1_all.py
@@ -79,12 +79,12 @@ def candidates_prep_function(all_candidates, n_selection=None):
 batch_size = 1
 
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
+
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)
@@ -267,7 +267,7 @@ def load_pretrained_model(l_in):
                 b=nn.init.Constant(0))
 
 
-    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/eavsteen/","r_fred_malignancy_7-20170404-163552.pkl"))
+    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/","r_fred_malignancy_7-20170404-163552.pkl"))
     nn.layers.set_all_param_values(l, metadata['param_values'])
 
     return l

diff --git a/configs_class_dsb/dsb_a_eliasq15_mal7_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq15_mal7_s5_p8a1_spl.py
@@ -78,10 +78,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.STAGE2_PIDS)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq1_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq1_mal2_s5_p8a1_all.py
@@ -70,12 +70,12 @@ def candidates_prep_function(all_candidates, n_selection=None):
 batch_size = 1
 
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
+train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq1_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq1_mal2_s5_p8a1_spl.py
@@ -69,10 +69,13 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 
-train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
+
 #In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
+train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq3_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq3_mal2_s5_p8a1_all.py
@@ -67,12 +67,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq3_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq3_mal2_s5_p8a1_spl.py
@@ -67,10 +67,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq4_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq4_mal2_s5_p8a1_all.py
@@ -79,12 +79,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 batch_size = 1
 
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq4_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq4_mal2_s5_p8a1_spl.py
@@ -78,10 +78,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq5_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq5_mal2_s5_p8a1_all.py
@@ -80,12 +80,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 
 
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq5_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq5_mal2_s5_p8a1_spl.py
@@ -77,10 +77,12 @@ def candidates_prep_function(all_candidates, n_selection=None):
 
 # data iterators
 batch_size = 1
+
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 

diff --git a/configs_class_dsb/dsb_a_eliasq6_mal2_s5_p8a1_all.py b/configs_class_dsb/dsb_a_eliasq6_mal2_s5_p8a1_all.py
@@ -69,12 +69,12 @@ def candidates_prep_function(all_candidates, n_selection=None):
 # data iterators
 batch_size = 1
 train_valid_ids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test']
+train_pids, valid_pids, test_pids, stage2_pids = train_valid_ids['training'], train_valid_ids['validation'], train_valid_ids['test'], train_valid_ids['test_stage2']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)
 print 'n test', len(test_pids)
 all_pids = train_pids + valid_pids + test_pids
-stage2_pids = [] #utils.load_pkl(pathfinder.STAGE2_PIDS)
+
 
 id2label = utils_lung.read_labels(pathfinder.LABELS_PATH)
 id2label_test = utils_lung.read_test_labels(pathfinder.TEST_LABELS_PATH)

diff --git a/configs_class_dsb/dsb_a_eliasq6_mal2_s5_p8a1_spl.py b/configs_class_dsb/dsb_a_eliasq6_mal2_s5_p8a1_spl.py
@@ -68,10 +68,11 @@ def candidates_prep_function(all_candidates, n_selection=None):
 
 # data iterators
 batch_size = 1
+d_pids = utils.load_pkl(pathfinder.VALIDATION_SPLIT_PATH)
+test_pids = d_pids['test_stage2']
+
 train_valid_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-#In the final stage we will have to add the pids of the new data
-#test_ids = utils.load_pkl(pathfinder.FINAL_SPLIT_PATH)
-train_pids, valid_pids, test_pids = train_valid_ids['train'], train_valid_ids['test'], [] #test_ids
+train_pids, valid_pids = train_valid_ids['train'], train_valid_ids['test']
 print 'n train', len(train_pids)
 print 'n valid', len(valid_pids)