output pred information of test

src/cmd.trigger.sh

@@ -1,11 +1,12 @@
 #!/bin/sh
 
 # feng
-CUDA_VISIBLE_DEVICES=0 python trigger_lstm.py -train ../ni_data/pre_processed_feng/tmp.train -test ../ni_data/pre_processed_feng/tmp.test  -tag ../ni_data/pre_processed_feng/labellist -pretrain_embed ../ni_data/pre_processed_feng/wordvector -vocab ../ni_data/pre_processed_feng/wordlist -model ../ni_data/models/model.trigger --batch_size 10 --lr 0.3 --no_use_pos --no_use_conv --dropout 0.2
+CUDA_VISIBLE_DEVICES=0 python trigger_lstm.py -train ../ni_data/pre_processed_feng/tmp.train -test ../ni_data/pre_processed_feng/tmp.test  -tag ../ni_data/pre_processed_feng/labellist -pretrain_embed ../ni_data/pre_processed_feng/wordvector -vocab ../ni_data/pre_processed_feng/wordlist -model ../ni_data/models/model.trigger --batch_size 10 --lr 0.5 --no_use_pos --no_use_conv --dropout 0.1 #--test_as_dev
+# --opti_flag adadelta
 
 #CUDA_VISIBLE_DEVICES=3 python trigger_lstm.py -train ../ni_data/pre_processed_feng/tmp.train -test ../ni_data/pre_processed_feng/tmp.test  -tag ../ni_data/pre_processed_feng/labellist -pretrain_embed ../ni_data/pre_processed_feng/wordvector -vocab ../ni_data/pre_processed_feng/wordlist -model ../ni_data/models/model.trigger
 
 # qin corpus
-#CUDA_VISIBLE_DEVICES=0 python trigger_lstm.py -train ../ni_data/EngMix/train.triggerEvent.txt -dev ../ni_data/EngMix/dev.triggerEvent.txt -test ../ni_data/EngMix/test.triggerEvent.txt -pretrain_embed ../ni_data/ace.pretrain300.vectors -model ../ni_data/models/model.trigger --batch_size 10 --lr 0.5 --no_use_pos --no_use_conv
+#CUDA_VISIBLE_DEVICES=0 python trigger_lstm.py -train ../ni_data/EngMix/train.triggerEvent.txt -dev ../ni_data/EngMix/dev.triggerEvent.txt -test ../ni_data/EngMix/test.triggerEvent.txt -pretrain_embed ../ni_data/ace.pretrain300.vectors -model ../ni_data/models/model.trigger --batch_size 10 --lr 0.03 --no_use_pos --no_use_conv --dropout 0.1
 
 #CUDA_VISIBLE_DEVICES=3 python trigger_lstm.py -train ../ni_data/EngMix/train.triggerEvent.txt -dev ../ni_data/EngMix/dev.triggerEvent.txt -test ../ni_data/EngMix/test.triggerEvent.txt -pretrain_embed ../ni_data/sskip.100.vectors -model ../ni_data/models/model.trigger

src/trigger_lstm.py

@@ -41,7 +41,7 @@ def arr2tensor(arr):
 def tensor2var(eg_tensor):
     return autograd.Variable(eg_tensor, requires_grad=False)
 
-def eval_model(data_loader, model, loss_function, data_flag, gpu, vocab=None):
+def eval_model(data_loader, model, loss_function, data_flag, gpu, vocab=None, tags_data=None):
     debug = False
     loss_all = 0
     gold_results = []
@@ -99,7 +99,7 @@ def eval_model(data_loader, model, loss_function, data_flag, gpu, vocab=None):
         for i in range(10):
             print i, gold_results[i]
             print i, pred_results[i]
-    prf = evalPRF(gold_results, pred_results, data_flag, debug_sents=debug_sents, vocab=vocab)
+    prf = evalPRF(gold_results, pred_results, data_flag, debug_sents=debug_sents, vocab=vocab, tags_data=tags_data)
     prf_iden = evalPRF_iden(gold_results, pred_results)
     #prf_iden2 = evalPRF_iden(gold_results, pred_results_iden)
     return loss_all, prf, prf_iden
@@ -116,7 +116,7 @@ def train_func(para_arr, args, data_sets, debug=False):
     model_path = para_arr[6]
     gpu = args.gpu
 
-    train_loader, dev_loader, test_loader, pretrain_embedding, vocab = data_sets
+    train_loader, dev_loader, test_loader, pretrain_embedding, vocab, tags_data = data_sets
 
     random_dim = 10
 
@@ -244,7 +244,7 @@ def train_func(para_arr, args, data_sets, debug=False):
         if epoch >= 2 and epoch % 10 == 0:
             if epoch % 50 == 0:
                 model_test = torch.load(model_path)
-                loss_test, prf_test, prf_test_iden = eval_model(test_loader, model_test, loss_function, "test_final", gpu, vocab=vocab)
+                loss_test, prf_test, prf_test_iden = eval_model(test_loader, model_test, loss_function, "test_final", gpu, vocab=vocab, tags_data=tags_data)
                 model_test = None
             else:
                 loss_test, prf_test, prf_test_iden = eval_model(test_loader, model, loss_function, "test", gpu)
@@ -256,7 +256,8 @@ def train_func(para_arr, args, data_sets, debug=False):
 
 # final result on test
     model = torch.load(model_path)
-    loss_test, prf_test, prf_test_iden = eval_model(test_loader, model, loss_function, "test_final", gpu, vocab=vocab)
+    loss_train, prf_train, prf_train_iden = eval_model(train_loader, model, loss_function, "test_final", gpu, vocab=vocab, tags_data=tags_data)
+    loss_test, prf_test, prf_test_iden = eval_model(test_loader, model, loss_function, "test_final", gpu, vocab=vocab, tags_data=tags_data)
     print "## Final results on test", loss_test, time.asctime(), Tab,
     outputPRF(prf_test)
     print "## Iden result:",
@@ -285,12 +286,12 @@ def train_func(para_arr, args, data_sets, debug=False):
             dev_data = training_data[-500:]
             print "first example of dev", dev_data[0]
     model_path = model_path + "_" + time.strftime("%Y%m%d%H%M%S", time.gmtime()) + "_"
-    #sys.exit(0)
 
     vocab_size = len(vocab)
     pretrain_vocab_size, pretrain_embed_dim = pretrain_embedding.shape
     tagset_size = len(tags_data)
 
+    #sys.exit(0)
     if 0:
         all_data = training_data+dev_data+test_data
         sent_lens = [len(item[0]) for item in all_data]
@@ -342,7 +343,7 @@ def train_func(para_arr, args, data_sets, debug=False):
 
     #dev_loader = torch_data.DataLoader(train_dataset, batch_size=args.batch_size)
     #test_loader = None
-    data_sets = train_loader, dev_loader, test_loader, pretrain_embedding, vocab
+    data_sets = train_loader, dev_loader, test_loader, pretrain_embedding, vocab, tags_data
 
     # begin to train
     train_func(para_arr, args, data_sets)

src/util.py

@@ -245,7 +245,13 @@ def eval_sysResult(sys_out, gold_outs, eval_flag="class"):
     if len(matched) == 0: return None
     return matched[0]
 
-def evalPRF(items_in_docs_gold, items_in_docs, data_flag="train", debug_sents=None, vocab=None):
+def evalPRF(items_in_docs_gold, items_in_docs, data_flag="train", debug_sents=None, vocab=None, tags_data=None):
+    if data_flag == "test_final": id2tagname = dict([(tags_data[item], item) for item in tags_data])
+    if data_flag == "test_final": 
+        gold_tags = []
+        pred_tags = []
+        pred_tags_missed = []
+        pred_tags_wrong = []
     debug = False
     common_in_docs = []
     num_in_docs_gold = []
@@ -257,13 +263,19 @@ def evalPRF(items_in_docs_gold, items_in_docs, data_flag="train", debug_sents=No
         missed_in_doc_gold = [item for item in items_in_doc_gold if item not in common_in_doc]
         wrong_in_doc = [item for item in items_in_doc if item not in common_in_doc]
         if data_flag == "test_final":
-            print "## final results of test doc:", len(common_in_docs)+1, " common, wrong, missed", common_in_doc, wrong_in_doc, missed_in_doc_gold
-        if data_flag == "test_final":
+            gold_tags.extend([item[1] for item in items_in_doc_gold])
+            pred_tags.extend([item[1] for item in items_in_doc])
+            pred_tags_missed.extend([item[1] for item in missed_in_doc_gold])
+            pred_tags_wrong.extend([item[1] for item in wrong_in_doc])
+            debug_words = idsent2words(debug_sent, vocab)
+            print "## eval debug sent", doc_id, zip(debug_words, range(len(debug_sent)))
+            if len(items_in_doc_gold) > 0:
+            #    print "## final results of test doc:", len(common_in_docs)+1, " common, wrong, missed", common_in_doc, wrong_in_doc, missed_in_doc_gold
+                print "## gold trig", [(item[0], debug_words[item[0]], item[1], id2tagname.get(item[1])) for item in items_in_doc_gold]
             if len(missed_in_doc_gold)+len(wrong_in_doc) > 0:
-                debug_words = idsent2words(debug_sent, vocab)
-                print "## eval debug sent", doc_id, debug_words
-                print "## missed trig", [(debug_words[item[0]], item[1]) for item in missed_in_doc_gold]
-                print "## wrong trig", [(debug_words[item[0]], item[1]) for item in missed_in_doc_gold]
+                if len(missed_in_doc_gold) > 0: print "## missed trig", [(item[0], debug_words[item[0]], item[1], id2tagname.get(item[1])) for item in missed_in_doc_gold], "\t\t",
+                if len(wrong_in_doc)>0: print "## wrong trig", [(item[0], debug_words[item[0]], item[1], id2tagname.get(item[1])) for item in wrong_in_doc],
+                print
 
         common_in_docs.append(len(common_in_doc))
         num_in_docs_gold.append(len(items_in_doc_gold))
@@ -280,6 +292,15 @@ def evalPRF(items_in_docs_gold, items_in_docs, data_flag="train", debug_sents=No
         #print "-- common_in_docs", common_in_docs
         #print "-- num_in_docs_gold", num_in_docs_gold
         #print "-- num_in_docs", num_in_docs
+        pred_tags_counter = Counter(pred_tags)
+        pred_tags_missed_counter = Counter(pred_tags_missed)
+        pred_tags_wrong_counter  = Counter(pred_tags_wrong)
+        print "## tag pred distribution: tag    num     ratio   pred_num    pred_ratio      missed_num      missed_ratio     wrong_num      wrong_ratio"
+        for item in Counter(gold_tags).most_common(): 
+            print id2tagname[item[0]], "\t\t\t", item[1], "\t", round(item[1]*100.0/len(gold_tags)), "\t",
+            print pred_tags_counter[item[0]], "\t", round(pred_tags_counter[item[0]]*100.0/len(pred_tags)), "\t",
+            print pred_tags_missed_counter[item[0]], "\t", round(pred_tags_missed_counter[item[0]]*100.0/len(pred_tags_missed)), "\t",
+            print pred_tags_wrong_counter[item[0]], "\t", round(pred_tags_wrong_counter[item[0]]*100.0/len(pred_tags_wrong))
 
     pre, rec, f1 = calPRF(common, num, num_gold)
     return pre, rec, f1
@@ -363,12 +384,18 @@ def resizeVocab(data_arr, vocab, pretrain_embedding):
             words_change[i] = len(words_change)
     new_data_arr = []
     for data in data_arr:
-        data = [([words_change[word] if word in words_top else new_vocab_size-1 for word in sent], tag) for sent, tag in data]
-        new_data_arr.append(data)
+        #print "## before data vocab change"
+        #for sent, tag in data[:10]: print zip(sent, idsent2words(sent, vocab))
+        new_data = [([words_change[word] if word in words_top else new_vocab_size-1 for word in sent], tag) for sent, tag in data]
+        new_data_arr.append(new_data)
     #print len(words_top)
     words_del = [i for i in range(old_vocab_size) if i not in words_top]
     pretrain_embedding = np.delete(pretrain_embedding, words_del, 0)
     vocab = dict([(wstr, words_change[wid]) for wstr, wid in vocab.items() if wid in words_top])
+    #for data in new_data_arr:
+    #    print "## aft data vocab change"
+    #    for sent, tag in data[:10]: print zip(sent, idsent2words(sent, vocab))
+    #    break
     return new_data_arr, vocab, pretrain_embedding
 
 def sort_data(dataset):
@@ -397,7 +424,9 @@ def load_data(args):
     #test_data = check_data(test_data, vocab)
     #check_trigger_test(training_data, test_data)
 
+    id2word = dict([(word_index, word) for word, word_index in vocab.items()])
     [training_data, test_data], vocab, pretrain_embedding = resizeVocab([training_data, test_data], vocab, pretrain_embedding)
+    id2word = dict([(word_index, word) for word, word_index in vocab.items()])
 
 # tags_data: tag_name: tag_id
     tags_data = loadTag(args.tag)