calculate_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adb2479b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import argparse\n",
    "import numpy as np\n",
    "import pytrec_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "91ba5e15",
   "metadata": {},
   "outputs": [],
   "source": [
    "def question_breakdown(result_path, qrel_path, subdomain_path):\n",
    "    all_result_ori = json.load(open(result_path, \"r\"))\n",
    "    all_result = {}\n",
    "    for did in all_result_ori:\n",
    "        new_did = \"_\".join(did.split(\"_\")[-2:])\n",
    "        all_result[new_did] = all_result_ori[did]\n",
    "    \n",
    "    qrels = json.load(open(qrel_path, \"r\"))\n",
    "    subdomain = json.load(open(subdomain_path, \"r\"))\n",
    "    subdomain[\"all\"] = list(all_result.keys())\n",
    "    \n",
    "    print(result_path + \"\\n\")\n",
    "    metric_results = None\n",
    "    for source, pids in subdomain.items():\n",
    "        sqrels = dict(filter(lambda x: x[0] in pids, qrels.items()))\n",
    "        sqrels = dict(filter(lambda x: x[1] != {\"\": 1}, sqrels.items())) # QReCC: filtering missings\n",
    "        sresults = dict(filter(lambda x: x[0] in pids, all_result.items()))\n",
    "\n",
    "        evaluator = pytrec_eval.RelevanceEvaluator(\n",
    "                sqrels, {\"recip_rank\", \"recall\", \"map\", \"ndcg\"})\n",
    "        metrics = evaluator.evaluate(sresults)\n",
    "        mrr_list = [v[\"recip_rank\"] for v in metrics.values()]\n",
    "        recall_5_list = [v[\"recall_5\"] for v in metrics.values()]\n",
    "        recall_10_list = [v[\"recall_10\"] for v in metrics.values()]\n",
    "        recall_20_list = [v[\"recall_20\"] for v in metrics.values()]\n",
    "        recall_30_list = [v[\"recall_30\"] for v in metrics.values()]\n",
    "        recall_100_list = [v[\"recall_100\"] for v in metrics.values()]\n",
    "        map_list = [v[\"map\"] for v in metrics.values()]\n",
    "        ndcg_list = [v[\"ndcg\"] for v in metrics.values()]\n",
    "        \n",
    "        np.set_printoptions(precision=4)\n",
    "        \n",
    "        eval_metrics = {\n",
    "            \"Recall@5\": round(100*np.average(recall_5_list), 2),\n",
    "            \"Recall@10\": round(100*np.average(recall_10_list), 2),\n",
    "            \"Recall@20\": round(100*np.average(recall_20_list), 2),\n",
    "            \"Recall@30\": round(100*np.average(recall_30_list), 2),\n",
    "            \"Recall@100\": round(100*np.average(recall_100_list), 2),\n",
    "            \"MRR\": round(100*np.average(mrr_list), 2),\n",
    "            \"map\": round(100*np.average(map_list), 2),\n",
    "            \"ndcg\": round(100*np.average(ndcg_list), 2),\n",
    "        }\n",
    "        print(source, len(sqrels))\n",
    "        print(eval_metrics)\n",
    "        print(\"\")\n",
    "        if source == 'all':\n",
    "            metric_results = metrics\n",
    "    return metric_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b73680a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "qrel_path = 'datasets/qrecc/qrels_test.txt'\n",
    "qtype_path = 'datasets/qrecc/test_question_types.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e8d9b60d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outputs/BM25/test_rewrite_bm25_scores.json\n",
      "\n",
      "trec 371\n",
      "{'Recall@5': 38.54, 'Recall@10': 53.77, 'Recall@20': 69.95, 'Recall@30': 76.82, 'Recall@100': 98.92, 'MRR': 27.34, 'map': 27.04, 'ndcg': 41.57}\n",
      "\n",
      "quac 6396\n",
      "{'Recall@5': 51.21, 'Recall@10': 62.9, 'Recall@20': 74.78, 'Recall@30': 81.1, 'Recall@100': 98.35, 'MRR': 40.32, 'map': 38.98, 'ndcg': 51.68}\n",
      "\n",
      "nq 1442\n",
      "{'Recall@5': 51.65, 'Recall@10': 63.8, 'Recall@20': 75.61, 'Recall@30': 81.69, 'Recall@100': 98.96, 'MRR': 40.78, 'map': 39.05, 'ndcg': 52.27}\n",
      "\n",
      "no-switch 279\n",
      "{'Recall@5': 68.57, 'Recall@10': 76.89, 'Recall@20': 87.03, 'Recall@30': 90.01, 'Recall@100': 100.0, 'MRR': 55.14, 'map': 53.58, 'ndcg': 64.2}\n",
      "\n",
      "switch 573\n",
      "{'Recall@5': 43.71, 'Recall@10': 58.98, 'Recall@20': 73.38, 'Recall@30': 80.47, 'Recall@100': 98.85, 'MRR': 34.23, 'map': 32.76, 'ndcg': 47.02}\n",
      "\n",
      "first 267\n",
      "{'Recall@5': 40.45, 'Recall@10': 56.65, 'Recall@20': 71.0, 'Recall@30': 79.21, 'Recall@100': 100.0, 'MRR': 32.7, 'map': 31.09, 'ndcg': 45.72}\n",
      "\n",
      "all 8209\n",
      "{'Recall@5': 50.71, 'Recall@10': 62.65, 'Recall@20': 74.71, 'Recall@30': 81.01, 'Recall@100': 98.48, 'MRR': 39.81, 'map': 38.45, 'ndcg': 51.33}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "temp_path = 'outputs/BM25/test_rewrite_bm25_scores.json'\n",
    "res = question_breakdown(temp_path, qrel_path, qtype_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "92f48184",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outputs/BM25/test_original_fused_ICL_bm25.json\n",
      "\n",
      "trec 371\n",
      "{'Recall@5': 14.82, 'Recall@10': 22.1, 'Recall@20': 29.51, 'Recall@30': 32.35, 'Recall@100': 42.99, 'MRR': 10.3, 'map': 10.27, 'ndcg': 16.83}\n",
      "\n",
      "quac 6396\n",
      "{'Recall@5': 12.1, 'Recall@10': 15.2, 'Recall@20': 18.99, 'Recall@30': 21.31, 'Recall@100': 27.7, 'MRR': 9.29, 'map': 8.84, 'ndcg': 12.8}\n",
      "\n",
      "nq 1442\n",
      "{'Recall@5': 11.14, 'Recall@10': 15.14, 'Recall@20': 18.36, 'Recall@30': 21.1, 'Recall@100': 29.06, 'MRR': 9.06, 'map': 8.64, 'ndcg': 12.88}\n",
      "\n",
      "no-switch 279\n",
      "{'Recall@5': 4.48, 'Recall@10': 5.38, 'Recall@20': 7.17, 'Recall@30': 7.53, 'Recall@100': 13.44, 'MRR': 3.25, 'map': 3.07, 'ndcg': 5.06}\n",
      "\n",
      "switch 573\n",
      "{'Recall@5': 7.41, 'Recall@10': 10.73, 'Recall@20': 14.92, 'Recall@30': 16.7, 'Recall@100': 23.72, 'MRR': 5.86, 'map': 5.75, 'ndcg': 9.37}\n",
      "\n",
      "first 267\n",
      "{'Recall@5': 40.45, 'Recall@10': 56.65, 'Recall@20': 71.0, 'Recall@30': 79.21, 'Recall@100': 100.0, 'MRR': 32.7, 'map': 31.09, 'ndcg': 45.72}\n",
      "\n",
      "all 8209\n",
      "{'Recall@5': 12.06, 'Recall@10': 15.5, 'Recall@20': 19.36, 'Recall@30': 21.77, 'Recall@100': 28.63, 'MRR': 9.3, 'map': 8.87, 'ndcg': 13.0}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "temp_path = 'outputs/BM25/test_original_fused_ICL_bm25.json'\n",
    "res = question_breakdown(temp_path, qrel_path, qtype_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "95ede350",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outputs/BM25/test_GPT_rewrite_ICL_post_bm25_scores.json\n",
      "\n",
      "trec 371\n",
      "{'Recall@5': 27.36, 'Recall@10': 39.89, 'Recall@20': 49.82, 'Recall@30': 53.59, 'Recall@100': 69.05, 'MRR': 19.02, 'map': 18.86, 'ndcg': 29.08}\n",
      "\n",
      "quac 6396\n",
      "{'Recall@5': 59.59, 'Recall@10': 68.28, 'Recall@20': 76.21, 'Recall@30': 80.32, 'Recall@100': 89.86, 'MRR': 49.81, 'map': 48.38, 'ndcg': 57.67}\n",
      "\n",
      "nq 1442\n",
      "{'Recall@5': 50.85, 'Recall@10': 60.13, 'Recall@20': 69.58, 'Recall@30': 74.69, 'Recall@100': 86.56, 'MRR': 41.51, 'map': 39.71, 'ndcg': 50.27}\n",
      "\n",
      "no-switch 279\n",
      "{'Recall@5': 80.74, 'Recall@10': 84.75, 'Recall@20': 88.83, 'Recall@30': 91.36, 'Recall@100': 94.89, 'MRR': 68.89, 'map': 67.42, 'ndcg': 74.11}\n",
      "\n",
      "switch 573\n",
      "{'Recall@5': 34.63, 'Recall@10': 44.48, 'Recall@20': 56.63, 'Recall@30': 63.18, 'Recall@100': 76.56, 'MRR': 25.29, 'map': 24.06, 'ndcg': 35.43}\n",
      "\n",
      "first 267\n",
      "{'Recall@5': 40.45, 'Recall@10': 56.65, 'Recall@20': 71.0, 'Recall@30': 79.21, 'Recall@100': 100.0, 'MRR': 32.7, 'map': 31.09, 'ndcg': 45.72}\n",
      "\n",
      "all 8209\n",
      "{'Recall@5': 56.6, 'Recall@10': 65.57, 'Recall@20': 73.85, 'Recall@30': 78.12, 'Recall@100': 88.34, 'MRR': 46.96, 'map': 45.53, 'ndcg': 55.08}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "temp_path = 'outputs/BM25/test_GPT_rewrite_ICL_post_bm25_scores.json'\n",
    "res = question_breakdown(temp_path, qrel_path, qtype_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d99532ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outputs/BM25/test_GPT_rewrite_fused_ZSL_post_bm25.json\n",
      "\n",
      "trec 371\n",
      "{'Recall@5': 26.82, 'Recall@10': 35.58, 'Recall@20': 47.12, 'Recall@30': 54.54, 'Recall@100': 72.15, 'MRR': 18.5, 'map': 18.26, 'ndcg': 29.1}\n",
      "\n",
      "quac 6396\n",
      "{'Recall@5': 55.02, 'Recall@10': 63.2, 'Recall@20': 71.39, 'Recall@30': 75.7, 'Recall@100': 85.55, 'MRR': 45.43, 'map': 44.11, 'ndcg': 53.31}\n",
      "\n",
      "nq 1442\n",
      "{'Recall@5': 45.3, 'Recall@10': 54.69, 'Recall@20': 64.06, 'Recall@30': 68.73, 'Recall@100': 81.92, 'MRR': 36.43, 'map': 34.81, 'ndcg': 45.31}\n",
      "\n",
      "no-switch 279\n",
      "{'Recall@5': 67.51, 'Recall@10': 73.14, 'Recall@20': 80.47, 'Recall@30': 84.71, 'Recall@100': 90.24, 'MRR': 57.47, 'map': 56.37, 'ndcg': 64.23}\n",
      "\n",
      "switch 573\n",
      "{'Recall@5': 31.16, 'Recall@10': 40.79, 'Recall@20': 51.42, 'Recall@30': 56.9, 'Recall@100': 72.45, 'MRR': 23.08, 'map': 22.02, 'ndcg': 32.8}\n",
      "\n",
      "first 267\n",
      "{'Recall@5': 40.45, 'Recall@10': 56.65, 'Recall@20': 71.0, 'Recall@30': 79.21, 'Recall@100': 100.0, 'MRR': 32.7, 'map': 31.09, 'ndcg': 45.72}\n",
      "\n",
      "all 8209\n",
      "{'Recall@5': 52.04, 'Recall@10': 60.46, 'Recall@20': 69.01, 'Recall@30': 73.52, 'Recall@100': 84.31, 'MRR': 42.63, 'map': 41.31, 'ndcg': 50.81}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "temp_path = 'outputs/BM25/test_GPT_rewrite_fused_ZSL_post_bm25.json'\n",
    "res = question_breakdown(temp_path, qrel_path, qtype_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "03bf5607",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outputs/BM25/test_Editor_rewrite_fused_ICL_editor_post_bm25.json\n",
      "\n",
      "trec 371\n",
      "{'Recall@5': 26.28, 'Recall@10': 36.25, 'Recall@20': 47.04, 'Recall@30': 51.62, 'Recall@100': 66.49, 'MRR': 17.43, 'map': 17.08, 'ndcg': 27.15}\n",
      "\n",
      "quac 6396\n",
      "{'Recall@5': 62.55, 'Recall@10': 70.46, 'Recall@20': 77.57, 'Recall@30': 81.31, 'Recall@100': 89.95, 'MRR': 53.01, 'map': 51.52, 'ndcg': 60.22}\n",
      "\n",
      "nq 1442\n",
      "{'Recall@5': 50.89, 'Recall@10': 59.63, 'Recall@20': 69.28, 'Recall@30': 74.25, 'Recall@100': 85.72, 'MRR': 41.57, 'map': 39.69, 'ndcg': 50.1}\n",
      "\n",
      "no-switch 279\n",
      "{'Recall@5': 83.83, 'Recall@10': 89.04, 'Recall@20': 91.8, 'Recall@30': 93.6, 'Recall@100': 96.7, 'MRR': 74.82, 'map': 73.4, 'ndcg': 79.23}\n",
      "\n",
      "switch 573\n",
      "{'Recall@5': 32.16, 'Recall@10': 41.17, 'Recall@20': 53.49, 'Recall@30': 59.67, 'Recall@100': 72.16, 'MRR': 22.35, 'map': 20.97, 'ndcg': 32.12}\n",
      "\n",
      "first 267\n",
      "{'Recall@5': 40.45, 'Recall@10': 56.65, 'Recall@20': 71.0, 'Recall@30': 79.21, 'Recall@100': 100.0, 'MRR': 32.7, 'map': 31.09, 'ndcg': 45.72}\n",
      "\n",
      "all 8209\n",
      "{'Recall@5': 58.86, 'Recall@10': 67.01, 'Recall@20': 74.73, 'Recall@30': 78.73, 'Recall@100': 88.15, 'MRR': 49.39, 'map': 47.89, 'ndcg': 56.94}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "temp_path = 'outputs/BM25/test_Editor_rewrite_fused_ICL_editor_post_bm25.json'\n",
    "res = question_breakdown(temp_path, qrel_path, qtype_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "660518ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outputs/BM25/test_Editor_rewrite_post_bm25_scores.json\n",
      "\n",
      "trec 371\n",
      "{'Recall@5': 32.08, 'Recall@10': 43.26, 'Recall@20': 55.12, 'Recall@30': 60.78, 'Recall@100': 77.18, 'MRR': 21.04, 'map': 20.79, 'ndcg': 32.34}\n",
      "\n",
      "quac 6396\n",
      "{'Recall@5': 60.49, 'Recall@10': 68.84, 'Recall@20': 77.01, 'Recall@30': 81.01, 'Recall@100': 91.09, 'MRR': 50.67, 'map': 49.18, 'ndcg': 58.55}\n",
      "\n",
      "nq 1442\n",
      "{'Recall@5': 51.88, 'Recall@10': 60.67, 'Recall@20': 70.27, 'Recall@30': 74.93, 'Recall@100': 87.97, 'MRR': 42.69, 'map': 40.64, 'ndcg': 51.34}\n",
      "\n",
      "no-switch 279\n",
      "{'Recall@5': 79.05, 'Recall@10': 82.35, 'Recall@20': 87.09, 'Recall@30': 89.67, 'Recall@100': 95.27, 'MRR': 69.76, 'map': 68.4, 'ndcg': 74.86}\n",
      "\n",
      "switch 573\n",
      "{'Recall@5': 38.08, 'Recall@10': 47.3, 'Recall@20': 59.63, 'Recall@30': 64.78, 'Recall@100': 78.88, 'MRR': 26.92, 'map': 25.21, 'ndcg': 37.0}\n",
      "\n",
      "first 267\n",
      "{'Recall@5': 40.45, 'Recall@10': 56.65, 'Recall@20': 71.0, 'Recall@30': 79.21, 'Recall@100': 100.0, 'MRR': 32.7, 'map': 31.09, 'ndcg': 45.72}\n",
      "\n",
      "all 8209\n",
      "{'Recall@5': 57.69, 'Recall@10': 66.25, 'Recall@20': 74.84, 'Recall@30': 79.03, 'Recall@100': 89.91, 'MRR': 47.93, 'map': 46.4, 'ndcg': 56.1}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "temp_path = 'outputs/BM25/test_Editor_rewrite_post_bm25_scores.json'\n",
    "res = question_breakdown(temp_path, qrel_path, qtype_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d9a2289",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}