llava/notebook/run_eval_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_RAD train: 314 3064 1241 1823 \n",
      "data_RAD test: 203 451 179 272 \n",
      "vqa_rad train: 313 1797 770 1027 \n",
      "vqa_rad test: 203 451 179 272 \n",
      "Slake1.0 train: 450 4919 2976 1943 \n",
      "Slake1.0 val: 96 1053 631 422 \n",
      "Slake1.0 test: 96 1061 645 416 \n",
      "pvqa train: 2599 19755 9949 9806 \n",
      "pvqa val: 832 6279 3144 3135 \n",
      "pvqa test: 858 6761 3370 3391 \n"
     ]
    }
   ],
   "source": [
    "import json, os\n",
    "import sys, tabulate\n",
    "\n",
    "\n",
    "dataset_list = [\"data_RAD\",\"vqa_rad\", \"Slake1.0\", \"pvqa\"]\n",
    "split_list = [\"train\", \"val\", \"test\"]\n",
    "for ds in dataset_list:\n",
    "    for sp in split_list:\n",
    "        split_file_path = f\"/home/chunyl/azure_mount/hanoverdev/clwon/llava/eval/{ds}/{sp}.json\"\n",
    "\n",
    "        if os.path.exists(split_file_path):\n",
    "            gt = json.load(open(split_file_path, 'r'))\n",
    "            # gt\n",
    "\n",
    "            # 'answer_type'\n",
    "            gt_imgs = list(set([item['image'] for item in gt]))\n",
    "            num_imgs = len(gt_imgs)\n",
    "\n",
    "            gt_ids = [item['id'] for item in gt]\n",
    "            num_ids = len(gt_ids)\n",
    "\n",
    "            num_open, num_close = 0, 0\n",
    "            for item in gt:\n",
    "                if item['answer_type'] == 'OPEN':\n",
    "                    num_open += 1\n",
    "                elif item['answer_type'] == 'CLOSED': \n",
    "                    num_close += 1\n",
    "                    \n",
    "            \n",
    "\n",
    "            print(f\"{ds} {sp}: {num_imgs} {num_ids} {num_open} {num_close} \")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 300648006,\n",
       " 'image': 'test/test_0648.jpg',\n",
       " 'answer_type': 'OPEN',\n",
       " 'question_type': 'where',\n",
       " 'conversations': [{'from': 'human', 'value': 'Where is this?\\n<image>'},\n",
       "  {'from': 'gpt', 'value': 'urinary'}]}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gt[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/data/llava/models/med-pretrain-364m-v1-1epoch/eval/zero-shot/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/data/llava/models/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/data/llava/models/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/vqa_rad-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/vqa_rad-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/finetune_llava_results/eval/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-15epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-15epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-15epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-v1-1epoch/eval/zero-shot/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-18epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-18epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-6epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-15epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-6epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-12epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-15epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-18epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-6epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-12epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-18epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-12epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-15epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-6epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zeroshot_to_downstream_results/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zeroshot_to_downstream_results/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/tune_projection_on_instruct-3epoch-2e-3/zeroshot_to_downstream_results/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n"
     ]
    }
   ],
   "source": [
    "pred_list_file = \"/home/chunyl/research/project/2023/llava_biomed/LLaVA_v1/eval_results_med_datasets.jsonl\"\n",
    "\n",
    "pred_list = open(pred_list_file, \"r\")\n",
    "\n",
    "pred_dict_list = []\n",
    "for pred in pred_list:\n",
    "    # print(pred)\n",
    "    pred_dict = json.loads(pred)\n",
    "    pred_dict_list.append(pred_dict)\n",
    "\n",
    "    print(pred_dict[\"pred_file\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "metric_list = ['exact match score', 'f1 score', 'precision', 'recall', 'open accuracy', 'yes/no accuracy'] \n",
    "\n",
    "def get_metrics(dataset='Slake1.0'):\n",
    "\n",
    "    dict_eval_ds= {}\n",
    "    for pred_dict in pred_dict_list:\n",
    "        if pred_dict['dataset'] == dataset:\n",
    "\n",
    "            string_data = pred_dict['results']\n",
    "            # split the string into rows  \n",
    "            rows = string_data.split(\"\\n\")  \n",
    "            \n",
    "            metric_dict = {}\n",
    "            for r in rows[:-1]:\n",
    "                for m in metric_list:\n",
    "                    if m in r:\n",
    "                        metric_dict[m] = float( r.split(m)[-1] )\n",
    "\n",
    "            json_path = \"/\".join(pred_dict[\"pred_file\"].split('/')[5:])\n",
    "            print(json_path)\n",
    "            print(metric_dict,\"\\n\")\n",
    "\n",
    "            dict_eval_ds[json_path] = metric_dict\n",
    "\n",
    "    return dict_eval_ds\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava-med/data/llava/models/med-pretrain-364m-v1-1epoch/eval/zero-shot/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 1.80001, 'f1 score': 3.31813, 'precision': 1.97184, 'recall': 13.7606, 'open accuracy': 7.26257, 'yes/no accuracy': 14.3382} \n",
      "\n",
      "llava-med/data/llava/models/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 14.3669, 'f1 score': 16.6764, 'precision': 14.3669, 'recall': 21.797, 'open accuracy': 16.7598, 'yes/no accuracy': 62.5} \n",
      "\n",
      "llava-med/data/llava/models/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 13.5475, 'f1 score': 16.03, 'precision': 13.5475, 'recall': 21.5829, 'open accuracy': 16.7598, 'yes/no accuracy': 58.4559} \n",
      "\n",
      "llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 3.75327, 'f1 score': 6.83915, 'precision': 4.22844, 'recall': 28.2409, 'open accuracy': 16.2011, 'yes/no accuracy': 59.5588} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.85663, 'f1 score': 8.4033, 'precision': 5.25715, 'recall': 30.117, 'open accuracy': 19.5531, 'yes/no accuracy': 59.1912} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/vqa_rad-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 20.9612, 'f1 score': 24.7495, 'precision': 21.1164, 'recall': 32.5791, 'open accuracy': 24.0223, 'yes/no accuracy': 63.9706} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 1.8939, 'f1 score': 3.45536, 'precision': 2.13359, 'recall': 15.2205, 'open accuracy': 8.37989, 'yes/no accuracy': 14.7059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/vqa_rad-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 16.8977, 'f1 score': 19.6958, 'precision': 16.8977, 'recall': 25.8333, 'open accuracy': 18.9944, 'yes/no accuracy': 67.6471} \n",
      "\n",
      "llava-med/llava-vicuna-7b/finetune_llava_results/eval/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 13.1338, 'f1 score': 15.078, 'precision': 13.1338, 'recall': 19.697, 'open accuracy': 13.9665, 'yes/no accuracy': 52.9412} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-v1-1epoch/eval/zero-shot/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 1.80001, 'f1 score': 3.31813, 'precision': 1.97184, 'recall': 13.7606, 'open accuracy': 7.26257, 'yes/no accuracy': 14.3382} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 2.05027, 'f1 score': 3.80405, 'precision': 2.33756, 'recall': 16.982, 'open accuracy': 9.49721, 'yes/no accuracy': 50.0} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 1.97599, 'f1 score': 3.87712, 'precision': 2.34433, 'recall': 19.595, 'open accuracy': 9.49721, 'yes/no accuracy': 47.4265} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 14.4094, 'f1 score': 16.9121, 'precision': 14.5025, 'recall': 21.8343, 'open accuracy': 14.5251, 'yes/no accuracy': 50.3676} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 10.8659, 'f1 score': 12.5774, 'precision': 10.8659, 'recall': 16.8436, 'open accuracy': 12.2905, 'yes/no accuracy': 56.9853} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.47139, 'f1 score': 7.42367, 'precision': 4.65307, 'recall': 23.9155, 'open accuracy': 12.2905, 'yes/no accuracy': 59.5588} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.84289, 'f1 score': 8.37923, 'precision': 5.17101, 'recall': 32.2137, 'open accuracy': 18.4358, 'yes/no accuracy': 57.3529} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-18epoch/test-answer-file.jsonl\n",
      "{'exact match score': 18.7443, 'f1 score': 22.4926, 'precision': 18.8375, 'recall': 30.4935, 'open accuracy': 22.905, 'yes/no accuracy': 64.7059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 15.5764, 'f1 score': 18.9018, 'precision': 15.6385, 'recall': 25.926, 'open accuracy': 19.5531, 'yes/no accuracy': 55.8824} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 19.5844, 'f1 score': 23.2847, 'precision': 19.6775, 'recall': 31.0335, 'open accuracy': 23.4637, 'yes/no accuracy': 64.7059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 12.1622, 'f1 score': 14.7204, 'precision': 12.1622, 'recall': 20.3208, 'open accuracy': 16.7598, 'yes/no accuracy': 55.8824} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-12epoch/test-answer-file.jsonl\n",
      "{'exact match score': 20.1007, 'f1 score': 24.3682, 'precision': 20.1628, 'recall': 33.4865, 'open accuracy': 26.257, 'yes/no accuracy': 62.8676} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-6epoch/test-answer-file.jsonl\n",
      "{'exact match score': 18.322, 'f1 score': 21.9643, 'precision': 18.322, 'recall': 29.9391, 'open accuracy': 24.0223, 'yes/no accuracy': 63.2353} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zeroshot_to_downstream_results/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 2.28586, 'f1 score': 4.00261, 'precision': 2.55415, 'recall': 13.3454, 'open accuracy': 7.82123, 'yes/no accuracy': 11.3971} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 1.69687, 'f1 score': 2.75613, 'precision': 1.79824, 'recall': 9.14565, 'open accuracy': 3.35196, 'yes/no accuracy': 13.2353} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 13.0074, 'f1 score': 15.3806, 'precision': 13.0074, 'recall': 20.4749, 'open accuracy': 13.9665, 'yes/no accuracy': 59.9265} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 3.95686, 'f1 score': 7.45899, 'precision': 4.52622, 'recall': 31.7747, 'open accuracy': 18.9944, 'yes/no accuracy': 60.6618} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 17.2255, 'f1 score': 20.1172, 'precision': 17.3054, 'recall': 26.4139, 'open accuracy': 18.4358, 'yes/no accuracy': 62.5} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 7.67691, 'f1 score': 9.13186, 'precision': 7.78864, 'recall': 12.2719, 'open accuracy': 11.1732, 'yes/no accuracy': 46.3235} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.37191, 'f1 score': 7.85362, 'precision': 4.90689, 'recall': 28.6509, 'open accuracy': 15.0838, 'yes/no accuracy': 56.6176} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.26453, 'f1 score': 7.69306, 'precision': 4.75565, 'recall': 29.5693, 'open accuracy': 16.7598, 'yes/no accuracy': 54.0441} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.05287, 'f1 score': 7.32268, 'precision': 4.52441, 'recall': 28.5082, 'open accuracy': 14.5251, 'yes/no accuracy': 59.1912} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zeroshot_to_downstream_results/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.27653, 'f1 score': 7.19277, 'precision': 4.58199, 'recall': 22.5235, 'open accuracy': 8.93855, 'yes/no accuracy': 55.5147} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.84115, 'f1 score': 7.68764, 'precision': 4.99642, 'recall': 24.8418, 'open accuracy': 11.1732, 'yes/no accuracy': 53.6765} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 15.3487, 'f1 score': 18.0523, 'precision': 15.3661, 'recall': 24.2081, 'open accuracy': 17.8771, 'yes/no accuracy': 63.9706} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 13.5475, 'f1 score': 16.03, 'precision': 13.5475, 'recall': 21.5829, 'open accuracy': 16.7598, 'yes/no accuracy': 58.4559} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 14.0968, 'f1 score': 16.3966, 'precision': 14.1527, 'recall': 21.5126, 'open accuracy': 15.0838, 'yes/no accuracy': 63.9706} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 13.0454, 'f1 score': 16.1067, 'precision': 13.1152, 'recall': 22.8864, 'open accuracy': 18.4358, 'yes/no accuracy': 55.1471} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 3.8488, 'f1 score': 7.23581, 'precision': 4.38255, 'recall': 30.2529, 'open accuracy': 16.2011, 'yes/no accuracy': 55.8824} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/vqa_rad-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 14.7446, 'f1 score': 17.0935, 'precision': 14.8244, 'recall': 22.0992, 'open accuracy': 16.2011, 'yes/no accuracy': 44.1176} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 17.5326, 'f1 score': 20.8123, 'precision': 17.5326, 'recall': 28.0354, 'open accuracy': 20.1117, 'yes/no accuracy': 51.8382} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 3.75327, 'f1 score': 6.83915, 'precision': 4.22844, 'recall': 28.2409, 'open accuracy': 16.2011, 'yes/no accuracy': 59.5588} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/tune_projection_on_instruct-3epoch-2e-3/zeroshot_to_downstream_results/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 0.819145, 'f1 score': 1.16587, 'precision': 0.819145, 'recall': 3.15667, 'open accuracy': 1.67598, 'yes/no accuracy': 7.72059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 2.65223, 'f1 score': 4.72556, 'precision': 2.97095, 'recall': 15.2594, 'open accuracy': 7.82123, 'yes/no accuracy': 15.0735} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_results/vqa_rad-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 1.75978, 'f1 score': 2.11138, 'precision': 1.75978, 'recall': 2.85847, 'open accuracy': 1.67598, 'yes/no accuracy': 13.6029} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 3.01722, 'f1 score': 4.7935, 'precision': 3.11724, 'recall': 16.2448, 'open accuracy': 8.37989, 'yes/no accuracy': 48.1618} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/vqa_rad/test-answer-file.jsonl\n",
      "{'exact match score': 4.43025, 'f1 score': 7.67947, 'precision': 4.79814, 'recall': 26.4034, 'open accuracy': 13.4078, 'yes/no accuracy': 56.9853} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "dict_eval_vqa_rad = get_metrics(dataset='vqa_rad')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.80739, 'f1 score': 8.307, 'precision': 5.14287, 'recall': 31.656, 'open accuracy': 17.3184, 'yes/no accuracy': 61.3971} \n",
      "\n",
      "llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 42.7086, 'f1 score': 50.8654, 'precision': 42.9647, 'recall': 64.581, 'open accuracy': 60.3352, 'yes/no accuracy': 77.9412} \n",
      "\n",
      "llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.35804, 'f1 score': 7.77379, 'precision': 4.81828, 'recall': 30.6541, 'open accuracy': 18.4358, 'yes/no accuracy': 55.5147} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 5.08654, 'f1 score': 8.74688, 'precision': 5.53446, 'recall': 31.7106, 'open accuracy': 16.7598, 'yes/no accuracy': 59.9265} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 41.8831, 'f1 score': 50.099, 'precision': 42.046, 'recall': 64.3948, 'open accuracy': 59.7765, 'yes/no accuracy': 81.9853} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 2.17797, 'f1 score': 4.00749, 'precision': 2.43355, 'recall': 16.148, 'open accuracy': 8.37989, 'yes/no accuracy': 21.3235} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 38.5333, 'f1 score': 46.1523, 'precision': 38.7584, 'recall': 59.3482, 'open accuracy': 53.6313, 'yes/no accuracy': 76.8382} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 6.25747, 'f1 score': 10.6424, 'precision': 6.68599, 'recall': 37.8374, 'open accuracy': 22.905, 'yes/no accuracy': 60.6618} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-15epoch/test-answer-file.jsonl\n",
      "{'exact match score': 42.8607, 'f1 score': 51.3, 'precision': 43.0237, 'recall': 65.6797, 'open accuracy': 63.1285, 'yes/no accuracy': 83.8235} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 43.0757, 'f1 score': 51.159, 'precision': 43.2387, 'recall': 64.7486, 'open accuracy': 62.0112, 'yes/no accuracy': 83.0882} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 1.90028, 'f1 score': 3.77613, 'precision': 2.23841, 'recall': 20.7392, 'open accuracy': 10.0559, 'yes/no accuracy': 59.1912} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 43.0624, 'f1 score': 51.2957, 'precision': 43.2254, 'recall': 65.3259, 'open accuracy': 60.3352, 'yes/no accuracy': 77.2059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 31.9744, 'f1 score': 38.4029, 'precision': 32.1473, 'recall': 50.0, 'open accuracy': 43.5754, 'yes/no accuracy': 65.0735} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 18.9572, 'f1 score': 22.2581, 'precision': 18.9572, 'recall': 29.9628, 'open accuracy': 23.4637, 'yes/no accuracy': 52.2059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.47224, 'f1 score': 7.53113, 'precision': 4.68687, 'recall': 27.0163, 'open accuracy': 13.9665, 'yes/no accuracy': 58.4559} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.07918, 'f1 score': 7.26433, 'precision': 4.50359, 'recall': 28.232, 'open accuracy': 16.2011, 'yes/no accuracy': 61.3971} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-6epoch/test-answer-file.jsonl\n",
      "{'exact match score': 40.1459, 'f1 score': 48.3958, 'precision': 40.464, 'recall': 62.7561, 'open accuracy': 58.1006, 'yes/no accuracy': 77.5735} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-15epoch/test-answer-file.jsonl\n",
      "{'exact match score': 40.6331, 'f1 score': 48.4466, 'precision': 40.8582, 'recall': 61.527, 'open accuracy': 57.5419, 'yes/no accuracy': 84.1912} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 43.2992, 'f1 score': 51.6042, 'precision': 43.4621, 'recall': 66.257, 'open accuracy': 60.3352, 'yes/no accuracy': 80.8824} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-18epoch/test-answer-file.jsonl\n",
      "{'exact match score': 40.5276, 'f1 score': 48.2673, 'precision': 40.7526, 'recall': 61.3687, 'open accuracy': 56.4246, 'yes/no accuracy': 81.25} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-12epoch/test-answer-file.jsonl\n",
      "{'exact match score': 39.2117, 'f1 score': 46.9121, 'precision': 39.4677, 'recall': 59.7858, 'open accuracy': 54.7486, 'yes/no accuracy': 80.8824} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 35.3789, 'f1 score': 42.5265, 'precision': 35.472, 'recall': 55.4988, 'open accuracy': 49.7207, 'yes/no accuracy': 66.5441} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 18.1937, 'f1 score': 21.5714, 'precision': 18.2868, 'recall': 28.6127, 'open accuracy': 22.3464, 'yes/no accuracy': 56.25} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 1.81193, 'f1 score': 2.7474, 'precision': 1.81193, 'recall': 8.05824, 'open accuracy': 6.14525, 'yes/no accuracy': 2.94118} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 2.28822, 'f1 score': 4.20376, 'precision': 2.60402, 'recall': 15.2689, 'open accuracy': 9.49721, 'yes/no accuracy': 12.5} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 14.8472, 'f1 score': 18.4038, 'precision': 15.052, 'recall': 26.8768, 'open accuracy': 17.3184, 'yes/no accuracy': 50.0} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.25412, 'f1 score': 7.4992, 'precision': 4.61965, 'recall': 30.509, 'open accuracy': 17.8771, 'yes/no accuracy': 61.7647} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 41.9258, 'f1 score': 49.7531, 'precision': 42.0888, 'recall': 63.3054, 'open accuracy': 59.2179, 'yes/no accuracy': 80.8824} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 34.1139, 'f1 score': 41.3113, 'precision': 34.2768, 'recall': 54.1155, 'open accuracy': 49.162, 'yes/no accuracy': 64.7059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 13.8379, 'f1 score': 16.3609, 'precision': 13.8379, 'recall': 22.631, 'open accuracy': 15.6425, 'yes/no accuracy': 58.0882} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.14003, 'f1 score': 7.36344, 'precision': 4.54431, 'recall': 27.9396, 'open accuracy': 16.2011, 'yes/no accuracy': 60.6618} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.29779, 'f1 score': 7.48774, 'precision': 4.67455, 'recall': 29.6701, 'open accuracy': 17.3184, 'yes/no accuracy': 61.3971} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.44032, 'f1 score': 7.59821, 'precision': 4.73902, 'recall': 29.8033, 'open accuracy': 17.8771, 'yes/no accuracy': 55.1471} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 5.03747, 'f1 score': 8.1, 'precision': 5.18848, 'recall': 25.7873, 'open accuracy': 15.6425, 'yes/no accuracy': 57.3529} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 41.2601, 'f1 score': 49.0988, 'precision': 41.423, 'recall': 62.8678, 'open accuracy': 59.7765, 'yes/no accuracy': 79.0441} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 12.8855, 'f1 score': 14.5366, 'precision': 12.8855, 'recall': 18.5901, 'open accuracy': 15.0838, 'yes/no accuracy': 55.5147} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 23.3489, 'f1 score': 27.4686, 'precision': 23.3997, 'recall': 36.3873, 'open accuracy': 29.0503, 'yes/no accuracy': 55.8824} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 3.57637, 'f1 score': 6.70502, 'precision': 4.10597, 'recall': 27.0162, 'open accuracy': 15.0838, 'yes/no accuracy': 61.0294} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 32.7064, 'f1 score': 39.5333, 'precision': 32.9459, 'recall': 51.6574, 'open accuracy': 46.9274, 'yes/no accuracy': 62.8676} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 16.7474, 'f1 score': 19.8731, 'precision': 16.7474, 'recall': 26.5177, 'open accuracy': 21.2291, 'yes/no accuracy': 52.2059} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 1.3644, 'f1 score': 2.28799, 'precision': 1.57763, 'recall': 6.82973, 'open accuracy': 3.35196, 'yes/no accuracy': 4.77941} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.62662, 'f1 score': 7.91059, 'precision': 4.91705, 'recall': 29.6692, 'open accuracy': 17.8771, 'yes/no accuracy': 60.2941} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 12.667, 'f1 score': 16.2402, 'precision': 12.8159, 'recall': 24.9805, 'open accuracy': 15.0838, 'yes/no accuracy': 51.8382} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 1.9773, 'f1 score': 3.69412, 'precision': 2.2414, 'recall': 15.3284, 'open accuracy': 8.37989, 'yes/no accuracy': 15.4412} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.22403, 'f1 score': 7.03075, 'precision': 4.41487, 'recall': 22.8672, 'open accuracy': 10.0559, 'yes/no accuracy': 55.1471} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.71841, 'f1 score': 8.03776, 'precision': 5.08746, 'recall': 28.4371, 'open accuracy': 17.3184, 'yes/no accuracy': 59.5588} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/data_RAD/test-answer-file.jsonl\n",
      "{'exact match score': 4.11485, 'f1 score': 6.98522, 'precision': 4.24731, 'recall': 26.9374, 'open accuracy': 13.9665, 'yes/no accuracy': 56.6176} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "dict_eval_data_RAD = get_metrics(dataset='data_RAD')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.82914, 'f1 score': 7.53261, 'precision': 4.38287, 'recall': 37.7117, 'open accuracy': 32.4031, 'yes/no accuracy': 49.7596} \n",
      "\n",
      "llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 47.9101, 'f1 score': 61.0086, 'precision': 48.4837, 'recall': 84.9668, 'open accuracy': 79.6899, 'yes/no accuracy': 85.5769} \n",
      "\n",
      "llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.27036, 'f1 score': 6.57152, 'precision': 3.78599, 'recall': 35.5258, 'open accuracy': 29.4574, 'yes/no accuracy': 53.8462} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.91481, 'f1 score': 7.54547, 'precision': 4.39161, 'recall': 38.0606, 'open accuracy': 31.938, 'yes/no accuracy': 50.9615} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 47.7918, 'f1 score': 60.8401, 'precision': 48.3081, 'recall': 84.8275, 'open accuracy': 80.0, 'yes/no accuracy': 83.6538} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 1.89119, 'f1 score': 3.59381, 'precision': 2.0422, 'recall': 21.9564, 'open accuracy': 19.6899, 'yes/no accuracy': 15.1442} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 46.6044, 'f1 score': 59.2743, 'precision': 47.0363, 'recall': 82.736, 'open accuracy': 77.5194, 'yes/no accuracy': 82.4519} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 4.06216, 'f1 score': 7.9052, 'precision': 4.58963, 'recall': 39.7282, 'open accuracy': 33.7984, 'yes/no accuracy': 54.3269} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-15epoch/test-answer-file.jsonl\n",
      "{'exact match score': 48.6179, 'f1 score': 61.9239, 'precision': 49.179, 'recall': 86.2345, 'open accuracy': 82.6357, 'yes/no accuracy': 87.0192} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 49.0431, 'f1 score': 62.4906, 'precision': 49.6426, 'recall': 87.1083, 'open accuracy': 83.5659, 'yes/no accuracy': 86.7788} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 2.48832, 'f1 score': 4.8154, 'precision': 2.73862, 'recall': 26.8206, 'open accuracy': 23.1008, 'yes/no accuracy': 50.2404} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 39.0815, 'f1 score': 49.4426, 'precision': 39.3193, 'recall': 69.0314, 'open accuracy': 62.6357, 'yes/no accuracy': 46.6346} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 44.0021, 'f1 score': 55.9572, 'precision': 44.4082, 'recall': 78.1761, 'open accuracy': 73.1783, 'yes/no accuracy': 63.2212} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 4.37319, 'f1 score': 7.97625, 'precision': 4.66502, 'recall': 34.3025, 'open accuracy': 29.9225, 'yes/no accuracy': 57.9327} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.90911, 'f1 score': 7.63191, 'precision': 4.43996, 'recall': 39.1717, 'open accuracy': 32.093, 'yes/no accuracy': 52.1635} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-6epoch/test-answer-file.jsonl\n",
      "{'exact match score': 46.0873, 'f1 score': 58.6322, 'precision': 46.5177, 'recall': 81.9355, 'open accuracy': 76.4341, 'yes/no accuracy': 82.2115} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-12epoch/test-answer-file.jsonl\n",
      "{'exact match score': 47.5911, 'f1 score': 60.5964, 'precision': 48.1783, 'recall': 84.2577, 'open accuracy': 79.0698, 'yes/no accuracy': 82.2115} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 39.6864, 'f1 score': 50.4951, 'precision': 40.1859, 'recall': 70.5764, 'open accuracy': 64.3411, 'yes/no accuracy': 54.5673} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 45.103, 'f1 score': 57.6148, 'precision': 45.6609, 'recall': 80.5698, 'open accuracy': 75.6589, 'yes/no accuracy': 64.1827} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-18epoch/test-answer-file.jsonl\n",
      "{'exact match score': 47.4204, 'f1 score': 60.4491, 'precision': 47.918, 'recall': 84.2378, 'open accuracy': 80.0, 'yes/no accuracy': 83.1731} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-15epoch/test-answer-file.jsonl\n",
      "{'exact match score': 47.102, 'f1 score': 59.8354, 'precision': 47.6447, 'recall': 83.0822, 'open accuracy': 78.2946, 'yes/no accuracy': 85.3365} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 46.5316, 'f1 score': 59.0818, 'precision': 46.993, 'recall': 82.2996, 'open accuracy': 77.5194, 'yes/no accuracy': 84.8558} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 2.67275, 'f1 score': 4.62001, 'precision': 2.85776, 'recall': 16.4678, 'open accuracy': 17.6744, 'yes/no accuracy': 6.73077} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 2.03511, 'f1 score': 3.7711, 'precision': 2.20234, 'recall': 18.548, 'open accuracy': 17.6744, 'yes/no accuracy': 13.4615} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 13.3915, 'f1 score': 17.8753, 'precision': 13.3925, 'recall': 29.1784, 'open accuracy': 29.9225, 'yes/no accuracy': 50.7212} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.15791, 'f1 score': 6.39037, 'precision': 3.63896, 'recall': 37.4844, 'open accuracy': 30.8527, 'yes/no accuracy': 51.4423} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 41.3632, 'f1 score': 52.2688, 'precision': 41.7441, 'recall': 72.7463, 'open accuracy': 66.8217, 'yes/no accuracy': 54.3269} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 44.5306, 'f1 score': 56.6606, 'precision': 44.8924, 'recall': 79.3349, 'open accuracy': 74.8837, 'yes/no accuracy': 64.9038} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.55989, 'f1 score': 7.01647, 'precision': 4.09815, 'recall': 34.5944, 'open accuracy': 26.5116, 'yes/no accuracy': 55.7692} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 4.15409, 'f1 score': 8.03834, 'precision': 4.69277, 'recall': 38.4412, 'open accuracy': 31.4729, 'yes/no accuracy': 52.4038} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.49015, 'f1 score': 7.04453, 'precision': 4.06387, 'recall': 38.0837, 'open accuracy': 32.4031, 'yes/no accuracy': 50.0} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.87237, 'f1 score': 6.91842, 'precision': 4.04417, 'recall': 31.4956, 'open accuracy': 28.2171, 'yes/no accuracy': 51.6827} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 40.9671, 'f1 score': 51.636, 'precision': 41.2496, 'recall': 71.6374, 'open accuracy': 66.0465, 'yes/no accuracy': 56.4904} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 44.3336, 'f1 score': 56.2828, 'precision': 44.7589, 'recall': 78.6042, 'open accuracy': 73.1783, 'yes/no accuracy': 63.4615} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.32768, 'f1 score': 6.8233, 'precision': 3.93311, 'recall': 36.7532, 'open accuracy': 31.0078, 'yes/no accuracy': 52.8846} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 40.244, 'f1 score': 50.8709, 'precision': 40.554, 'recall': 70.945, 'open accuracy': 64.9612, 'yes/no accuracy': 57.2115} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 16.2788, 'f1 score': 20.625, 'precision': 16.3753, 'recall': 28.9275, 'open accuracy': 30.5426, 'yes/no accuracy': 25.7212} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.00776, 'f1 score': 5.20786, 'precision': 3.18509, 'recall': 19.057, 'open accuracy': 19.0698, 'yes/no accuracy': 5.04808} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 3.27036, 'f1 score': 6.57152, 'precision': 3.78599, 'recall': 35.5258, 'open accuracy': 29.4574, 'yes/no accuracy': 53.8462} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 13.8454, 'f1 score': 18.4803, 'precision': 13.8579, 'recall': 30.3271, 'open accuracy': 28.2171, 'yes/no accuracy': 50.2404} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 2.25075, 'f1 score': 4.27335, 'precision': 2.4873, 'recall': 23.6058, 'open accuracy': 21.8605, 'yes/no accuracy': 15.3846} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 4.74288, 'f1 score': 8.46138, 'precision': 4.96825, 'recall': 36.1622, 'open accuracy': 30.2326, 'yes/no accuracy': 57.4519} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 2.77097, 'f1 score': 5.04207, 'precision': 2.95395, 'recall': 22.6322, 'open accuracy': 20.0, 'yes/no accuracy': 43.9904} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/Slake1.0/test-answer-file.jsonl\n",
      "{'exact match score': 4.64493, 'f1 score': 8.35013, 'precision': 4.86213, 'recall': 36.7963, 'open accuracy': 32.093, 'yes/no accuracy': 57.2115} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "dict_eval_slake = get_metrics(dataset='Slake1.0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.925277, 'f1 score': 2.31502, 'precision': 1.40977, 'recall': 11.3417, 'open accuracy': 5.01484, 'yes/no accuracy': 49.6314} \n",
      "\n",
      "llava-med/llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 22.3608, 'f1 score': 27.822, 'precision': 22.4283, 'recall': 38.8213, 'open accuracy': 35.2522, 'yes/no accuracy': 92.3916} \n",
      "\n",
      "llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.801299, 'f1 score': 2.29458, 'precision': 1.36786, 'recall': 11.7571, 'open accuracy': 4.62908, 'yes/no accuracy': 53.1996} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.07622, 'f1 score': 2.50325, 'precision': 1.5427, 'recall': 11.1124, 'open accuracy': 5.34125, 'yes/no accuracy': 49.3365} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 21.9113, 'f1 score': 27.1263, 'precision': 21.9501, 'recall': 37.7646, 'open accuracy': 34.3323, 'yes/no accuracy': 91.6544} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.0815, 'f1 score': 2.2585, 'precision': 1.49898, 'recall': 8.0671, 'open accuracy': 3.88724, 'yes/no accuracy': 19.4928} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava_med-vicuna-7b-v0-pretrain-364m-v1-1epoch/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 22.07, 'f1 score': 27.423, 'precision': 22.129, 'recall': 38.2579, 'open accuracy': 34.8071, 'yes/no accuracy': 91.4185} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.09491, 'f1 score': 2.55186, 'precision': 1.57422, 'recall': 11.6467, 'open accuracy': 5.13353, 'yes/no accuracy': 49.0711} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-15epoch/test-answer-file.jsonl\n",
      "{'exact match score': 23.0535, 'f1 score': 28.5863, 'precision': 23.114, 'recall': 39.7589, 'open accuracy': 36.4392, 'yes/no accuracy': 91.212} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 22.9572, 'f1 score': 28.4806, 'precision': 23.0221, 'recall': 39.6015, 'open accuracy': 36.0534, 'yes/no accuracy': 91.0941} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.577195, 'f1 score': 1.56126, 'precision': 0.95177, 'recall': 8.74171, 'open accuracy': 4.21365, 'yes/no accuracy': 45.6503} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 15.4927, 'f1 score': 19.1709, 'precision': 15.495, 'recall': 26.9196, 'open accuracy': 24.451, 'yes/no accuracy': 81.7753} \n",
      "\n",
      "llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 5.19793, 'f1 score': 5.98005, 'precision': 5.21524, 'recall': 7.73732, 'open accuracy': 6.97329, 'yes/no accuracy': 63.1967} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.895472, 'f1 score': 1.87507, 'precision': 1.18574, 'recall': 8.37941, 'open accuracy': 4.09496, 'yes/no accuracy': 57.3872} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.900736, 'f1 score': 2.4255, 'precision': 1.46781, 'recall': 12.3029, 'open accuracy': 5.13353, 'yes/no accuracy': 54.0549} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-18epoch/test-answer-file.jsonl\n",
      "{'exact match score': 21.9152, 'f1 score': 27.1841, 'precision': 21.9829, 'recall': 37.8844, 'open accuracy': 34.6884, 'yes/no accuracy': 91.389} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 6.75777, 'f1 score': 8.16966, 'precision': 6.76695, 'recall': 11.1652, 'open accuracy': 10.3561, 'yes/no accuracy': 59.1861} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 20.8273, 'f1 score': 25.7641, 'precision': 20.8702, 'recall': 35.8833, 'open accuracy': 32.6706, 'yes/no accuracy': 89.1477} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-15epoch/test-answer-file.jsonl\n",
      "{'exact match score': 22.026, 'f1 score': 27.3085, 'precision': 22.051, 'recall': 37.948, 'open accuracy': 35.0742, 'yes/no accuracy': 91.212} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-6epoch/test-answer-file.jsonl\n",
      "{'exact match score': 21.5741, 'f1 score': 26.6384, 'precision': 21.6085, 'recall': 36.9811, 'open accuracy': 33.7389, 'yes/no accuracy': 90.4453} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-9epoch/test-answer-file.jsonl\n",
      "{'exact match score': 21.8479, 'f1 score': 27.0813, 'precision': 21.9006, 'recall': 37.5876, 'open accuracy': 34.184, 'yes/no accuracy': 91.5364} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.35049, 'f1 score': 1.72116, 'precision': 1.52792, 'recall': 3.55878, 'open accuracy': 1.69139, 'yes/no accuracy': 3.56827} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.20933, 'f1 score': 2.12787, 'precision': 1.53673, 'recall': 6.25707, 'open accuracy': 3.1454, 'yes/no accuracy': 13.5063} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.86452, 'f1 score': 2.088, 'precision': 1.89965, 'recall': 3.32244, 'open accuracy': 1.63205, 'yes/no accuracy': 58.7732} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.729064, 'f1 score': 2.12583, 'precision': 1.26326, 'recall': 11.4588, 'open accuracy': 4.54006, 'yes/no accuracy': 52.079} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 14.1818, 'f1 score': 17.392, 'precision': 14.2043, 'recall': 24.1878, 'open accuracy': 22.0178, 'yes/no accuracy': 71.6013} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 10.4474, 'f1 score': 12.5307, 'precision': 10.4751, 'recall': 17.1754, 'open accuracy': 15.1929, 'yes/no accuracy': 71.3654} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.846929, 'f1 score': 2.12915, 'precision': 1.29548, 'recall': 10.33, 'open accuracy': 4.273, 'yes/no accuracy': 52.6688} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.927741, 'f1 score': 2.36567, 'precision': 1.43842, 'recall': 11.4088, 'open accuracy': 5.25223, 'yes/no accuracy': 56.2371} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.862202, 'f1 score': 2.33317, 'precision': 1.40622, 'recall': 11.7007, 'open accuracy': 4.98516, 'yes/no accuracy': 59.6579} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.985642, 'f1 score': 1.98982, 'precision': 1.27497, 'recall': 8.49114, 'open accuracy': 4.12463, 'yes/no accuracy': 59.6579} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 14.8674, 'f1 score': 18.2641, 'precision': 14.8783, 'recall': 25.5016, 'open accuracy': 23.1157, 'yes/no accuracy': 82.8664} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 19.7024, 'f1 score': 24.3601, 'precision': 19.7345, 'recall': 34.0228, 'open accuracy': 31.0682, 'yes/no accuracy': 86.936} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.763731, 'f1 score': 2.18527, 'precision': 1.3047, 'recall': 11.1792, 'open accuracy': 5.01484, 'yes/no accuracy': 58.2719} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test-answer-file.jsonl\n",
      "{'exact match score': 14.3877, 'f1 score': 17.7475, 'precision': 14.4059, 'recall': 24.8317, 'open accuracy': 22.3739, 'yes/no accuracy': 81.7458} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test-answer-file.jsonl\n",
      "{'exact match score': 18.872, 'f1 score': 23.2424, 'precision': 18.9058, 'recall': 32.3852, 'open accuracy': 29.2582, 'yes/no accuracy': 87.2604} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.2464, 'f1 score': 1.66449, 'precision': 1.42599, 'recall': 3.54132, 'open accuracy': 1.75074, 'yes/no accuracy': 4.24653} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.801299, 'f1 score': 2.29458, 'precision': 1.36786, 'recall': 11.7571, 'open accuracy': 4.62908, 'yes/no accuracy': 53.1996} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.61247, 'f1 score': 1.86058, 'precision': 1.64185, 'recall': 3.07727, 'open accuracy': 1.27596, 'yes/no accuracy': 56.9743} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.09515, 'f1 score': 1.88844, 'precision': 1.34173, 'recall': 6.3527, 'open accuracy': 3.35312, 'yes/no accuracy': 14.7449} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 1.07171, 'f1 score': 2.1869, 'precision': 1.39674, 'recall': 8.99068, 'open accuracy': 4.21365, 'yes/no accuracy': 56.1191} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.880301, 'f1 score': 1.52039, 'precision': 1.04859, 'recall': 5.3987, 'open accuracy': 2.67062, 'yes/no accuracy': 52.6688} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch/zero-shot-on-fixed-data/pvqa/test-answer-file.jsonl\n",
      "{'exact match score': 0.83519, 'f1 score': 1.71824, 'precision': 1.08993, 'recall': 7.4324, 'open accuracy': 3.41246, 'yes/no accuracy': 57.6821} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "dict_eval_pvqa = get_metrics(dataset='pvqa')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch\n",
      "26.94 & 56.62 & 36.80 & 57.21 & 7.43 & 57.68 & 40.45 \n",
      "med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data\n",
      "27.02 & 58.46 & 34.30 & 57.93 & 8.38 & 57.39 & 40.58 \n",
      "llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data\n",
      "20.74 & 59.19 & 26.82 & 50.24 & 8.74 & 45.65 & 35.23 \n",
      "llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\n",
      "29.96 & 52.21 & 69.03 & 46.63 & 26.92 & 81.78 & 51.09 \n",
      "llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\n",
      "50.00 & 65.07 & 78.18 & 63.22 & 7.74 & 63.20 & 54.57 \n",
      "med-pretrain-364m-1epoch/zero-shot-on-fixed-data\n",
      "15.27 & 12.50 & 18.55 & 13.46 & 6.26 & 13.51 & 13.26 \n",
      "med-pretrain-364m-3epoch/zero-shot-on-fixed-data\n",
      "15.33 & 15.44 & 23.61 & 15.38 & 6.35 & 14.74 & 15.14 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data\n",
      "25.79 & 57.35 & 31.50 & 51.68 & 8.49 & 59.66 & 39.08 \n",
      "med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data\n",
      "22.87 & 55.15 & 36.16 & 57.45 & 8.99 & 56.12 & 39.46 \n",
      "med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data\n",
      "26.88 & 50.00 & 29.18 & 50.72 & 3.32 & 58.77 & 36.48 \n",
      "med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data\n",
      "24.98 & 51.84 & 30.33 & 50.24 & 3.08 & 56.97 & 36.24 \n",
      "med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data\n",
      "28.44 & 59.56 & 22.63 & 43.99 & 5.40 & 52.67 & 35.45 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data\n",
      "27.02 & 61.03 & 36.75 & 52.88 & 11.18 & 58.27 & 41.19 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data\n",
      "30.51 & 61.76 & 37.48 & 51.44 & 11.46 & 52.08 & 40.79 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data\n",
      "29.80 & 55.15 & 38.08 & 50.00 & 11.70 & 59.66 & 40.73 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data\n",
      "29.67 & 60.29 & 35.53 & 53.85 & 11.76 & 53.20 & 40.72 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data\n",
      "29.67 & 61.40 & 38.44 & 52.40 & 11.41 & 56.24 & 41.59 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data\n",
      "28.23 & 61.40 & 39.17 & 52.16 & 12.30 & 54.05 & 41.22 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data\n",
      "27.94 & 60.66 & 34.59 & 55.77 & 10.33 & 52.67 & 40.33 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/*-1epoch\n",
      "26.52 & 52.21 & 70.94 & 57.21 & 24.83 & 81.75 & 52.24 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/*-3epoch\n",
      "51.66 & 62.87 & 28.93 & 25.72 & 32.39 & 87.26 & 48.14 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\n",
      "22.63 & 58.09 & 72.75 & 54.33 & 24.19 & 71.60 & 50.60 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\n",
      "54.12 & 64.71 & 79.33 & 64.90 & 17.18 & 71.37 & 58.60 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\n",
      "36.39 & 55.88 & 71.64 & 56.49 & 25.50 & 82.87 & 54.79 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\n",
      "18.59 & 55.51 & 78.60 & 63.46 & 34.02 & 86.94 & 56.19 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\n",
      "28.61 & 56.25 & 70.58 & 54.57 & 11.17 & 59.19 & 46.73 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\n",
      "55.50 & 66.54 & 80.57 & 64.18 & 35.88 & 89.15 & 65.30 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-9epoch\n",
      "66.26 & 80.88 & 82.30 & 84.86 & 37.59 & 91.54 & 73.90 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-12epoch\n",
      "59.79 & 80.88 & 84.26 & 82.21 & 76.78 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-15epoch\n",
      "61.53 & 84.19 & 83.08 & 85.34 & 37.95 & 91.21 & 73.88 \n",
      "med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-18epoch\n",
      "61.37 & 81.25 & 84.24 & 83.17 & 37.88 & 91.39 & 73.22 \n",
      "med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data\n",
      "37.84 & 60.66 & 39.73 & 54.33 & 11.65 & 49.07 & 42.21 \n",
      "med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-9epoch\n",
      "64.75 & 83.09 & 87.11 & 86.78 & 39.60 & 91.09 & 75.40 \n",
      "med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-15epoch\n",
      "65.68 & 83.82 & 86.23 & 87.02 & 39.76 & 91.21 & 75.62 \n",
      "llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data\n",
      "31.66 & 61.40 & 37.71 & 49.76 & 11.34 & 49.63 & 40.25 \n",
      "llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data\n",
      "64.58 & 77.94 & 84.97 & 85.58 & 38.82 & 92.39 & 74.05 \n"
     ]
    }
   ],
   "source": [
    "dict_eval_ds_list = [dict_eval_data_RAD, dict_eval_slake, dict_eval_pvqa]\n",
    "\n",
    "exp_list = [\n",
    "    \"llava-vicuna-7b/med-train_e2e-instruct-data-run-from-llava/ablation/no-stage1-3epoch\",\n",
    "    \"med-train_e2e-instruct-data-run-from-llava-p-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval/zero-shot-on-fixed-data\",\n",
    "    \"llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\",\n",
    "    \"llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\",\n",
    "    \"med-pretrain-364m-1epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-3epoch/finetune_e2e_on_instruct_run2-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_postprocess_cleaned_10k_qa_44k_with_image-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_postprocess_im_cleaned_10k_qa_48k_with_image-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-9epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/*-1epoch\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch/finetune_to_downstream_results_fixed_data/*-3epoch\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\", \n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\", \n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\",\n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-1epoch\",   \n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-3epoch\",   \n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-9epoch\",   \n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-12epoch\",   \n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-15epoch\",  \n",
    "    \"med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-18epoch\",  \n",
    "    \"med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-fixed-data\",\n",
    "    \"med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-9epoch\",\n",
    "    \"med-pretrain-1epoch-BiomedCLIP/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/finetune_to_downstream_results_fixed_data/*-15epoch\",\n",
    "    \"llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/zero-shot-on-fixed-data\",\n",
    "    \"llava-vicuna-13b/llava_med-llava-13b-v0-pretrain-364m-v1-1epoch_consolidate/finetune_e2e_on_instruct_caption_in_text_cleaned-60k/finetune_to_downstream_results_fixed_data\"\n",
    "]\n",
    "\n",
    "\n",
    "exp_results_list = [] \n",
    "for exp in exp_list:\n",
    "\n",
    "    exp_results = []\n",
    "    for dict_eval_ds in dict_eval_ds_list:\n",
    "        for key, metric_value in dict_eval_ds.items():\n",
    "            if \"*\" in exp and exp.split(\"*\")[0] in key and exp.split(\"*\")[1]+\"/test-answer-file\" in key:\n",
    "                # print(exp.split(\"*\"), key)\n",
    "                recall, accuracy = dict_eval_ds[key]['recall'], dict_eval_ds[key]['yes/no accuracy']\n",
    "                exp_results.append(recall)\n",
    "                exp_results.append(accuracy)\n",
    "            elif exp in key:\n",
    "                recall, accuracy = dict_eval_ds[key]['recall'], dict_eval_ds[key]['yes/no accuracy']\n",
    "                exp_results.append(recall)\n",
    "                exp_results.append(accuracy)\n",
    "\n",
    "    exp_results_list.append(exp_results)\n",
    "\n",
    "for exp, exp_results in zip(exp_list, exp_results_list):\n",
    "    print(exp)\n",
    "\n",
    "    res = ''\n",
    "    for r in exp_results:\n",
    "        res += f'{r:.2f} & '\n",
    "\n",
    "    avg = sum(exp_results)/len(exp_results)\n",
    "    res += f'{avg:.2f} '\n",
    "    \n",
    "    print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-data-w-options/data_RAD/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-data-w-options/Slake1.0/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-data-w-options/pvqa/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-data-w-options/data_RAD/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-data-w-options/Slake1.0/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-data-w-options/pvqa/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test_w_options-answer-file.jsonl\n",
      "/home/chunyl/azure_mount/chunyleu_output/llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test_w_options-answer-file.jsonl\n"
     ]
    }
   ],
   "source": [
    "pred_list_file = \"/home/chunyl/research/project/2023/llava_biomed/LLaVA_v1/eval_results_med_datasets_w_options.jsonl\"\n",
    "\n",
    "pred_list = open(pred_list_file, \"r\")\n",
    "\n",
    "pred_dict_list = []\n",
    "for pred in pred_list:\n",
    "    # print(pred)\n",
    "    pred_dict = json.loads(pred)\n",
    "    pred_dict_list.append(pred_dict)\n",
    "\n",
    "    print(pred_dict[\"pred_file\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-data-w-options/data_RAD/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 3.74267, 'f1 score': 7.11678, 'precision': 4.39986, 'recall': 29.2483, 'open accuracy': 7.82123, 'yes/no accuracy': 83.871} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-data-w-options/data_RAD/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 3.34648, 'f1 score': 6.15806, 'precision': 3.75533, 'recall': 26.5786, 'open accuracy': 8.37989, 'yes/no accuracy': 82.0225} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-3epoch/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 22.2098, 'f1 score': 26.2827, 'precision': 22.2098, 'recall': 34.6648, 'open accuracy': 24.581, 'yes/no accuracy': 90.2703} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/data_RAD-1epoch/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 15.6564, 'f1 score': 18.3054, 'precision': 15.6564, 'recall': 24.4083, 'open accuracy': 15.6425, 'yes/no accuracy': 88.8889} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-data-w-options/Slake1.0/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 4.84596, 'f1 score': 9.67785, 'precision': 5.7379, 'recall': 43.1442, 'open accuracy': 17.9845, 'yes/no accuracy': 79.6813} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-data-w-options/Slake1.0/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 3.46803, 'f1 score': 7.4333, 'precision': 4.2963, 'recall': 41.2602, 'open accuracy': 17.6744, 'yes/no accuracy': 79.2683} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-3epoch/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 43.3072, 'f1 score': 54.9204, 'precision': 43.7752, 'recall': 76.4622, 'open accuracy': 65.5814, 'yes/no accuracy': 79.1096} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/Slake1.0-1epoch/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 38.8457, 'f1 score': 49.1824, 'precision': 39.3108, 'recall': 68.4261, 'open accuracy': 58.4496, 'yes/no accuracy': 72.5806} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch/zero-shot-on-data-w-options/pvqa/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 0.992069, 'f1 score': 2.29967, 'precision': 1.4248, 'recall': 10.3994, 'open accuracy': 0.0296736, 'yes/no accuracy': 86.8547} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/zero-shot-on-data-w-options/pvqa/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 0.753906, 'f1 score': 2.05575, 'precision': 1.24775, 'recall': 10.615, 'open accuracy': 0.0, 'yes/no accuracy': 85.1224} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-3epoch/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 17.6239, 'f1 score': 22.1197, 'precision': 17.6411, 'recall': 31.2504, 'open accuracy': 0.237389, 'yes/no accuracy': 82.4405} \n",
      "\n",
      "llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch/finetune_to_downstream_results_fixed_data/pvqa-1epoch/test_w_options-answer-file.jsonl\n",
      "{'exact match score': 13.2527, 'f1 score': 16.6341, 'precision': 13.2724, 'recall': 23.6063, 'open accuracy': 0.207715, 'yes/no accuracy': 92.3191} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "dict_eval_pvqa = get_metrics(dataset='data_RAD')\n",
    "dict_eval_pvqa = get_metrics(dataset='Slake1.0')\n",
    "dict_eval_pvqa = get_metrics(dataset='pvqa')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_eval_vqa_rad\n",
    "dict_eval_data_RAD\n",
    "\n",
    "def get_evaluated_ckpts(dict_eval_vqa):\n",
    "    ckpts_eval = []\n",
    "    for k, v in dict_eval_vqa.items():\n",
    "        ckpt_path = '/'.join(k.split('/')[:-2])\n",
    "        ckpts_eval.append(ckpt_path)\n",
    "    return ckpts_eval\n",
    "\n",
    "\n",
    "ckpts_eval_vqa_rad = get_evaluated_ckpts(dict_eval_vqa_rad)\n",
    "ckpts_eval_data_RAD = get_evaluated_ckpts(dict_eval_data_RAD)\n",
    "\n",
    "ckpts_eval_vqa_rad = list(set(ckpts_eval_vqa_rad))\n",
    "ckpts_eval_data_RAD = list(set(ckpts_eval_data_RAD))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_prefix = \"finetune_to_downstream_results_fixed_data\"\n",
    "zs_prefix = \"zero-shot-on-fixed-data\"\n",
    "\n",
    "ckpts_eval_vqa_rad_ft, ckpts_eval_vqa_rad_zs = [], []\n",
    "for ckpt_path in ckpts_eval_vqa_rad:\n",
    "    if ft_prefix in ckpt_path:\n",
    "        ckpts_eval_vqa_rad_ft.append(os.path.dirname(ckpt_path))\n",
    "    if zs_prefix in ckpt_path:\n",
    "        ckpts_eval_vqa_rad_zs.append(os.path.dirname(ckpt_path))\n",
    "\n",
    "ckpts_eval_data_RAD_ft, ckpts_eval_data_RAD_zs = [], []\n",
    "for ckpt_path in ckpts_eval_data_RAD:\n",
    "    if ft_prefix in ckpt_path:\n",
    "        ckpts_eval_data_RAD_ft.append(os.path.dirname(ckpt_path))\n",
    "    if zs_prefix in ckpt_path:\n",
    "        ckpts_eval_data_RAD_zs.append(os.path.dirname(ckpt_path))\n",
    "\n",
    "ckpts_eval_data_RAD_ft_difference = list(set(ckpts_eval_vqa_rad_ft) - set(ckpts_eval_data_RAD_ft)) \n",
    "ckpts_eval_data_RAD_zs_difference = list(set(ckpts_eval_vqa_rad_zs) - set(ckpts_eval_data_RAD_zs)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch',\n",
       " 'llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpts_eval_data_RAD_ft_difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-1epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_cleaned-60k-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch/finetune_e2e_on_instruct-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-1epoch',\n",
       " 'llava-med/data/llava/models/finetune_e2e_on_instruct_cleaned-60k-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-1epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct_caption_in_text_cleaned-60k-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-1epoch/finetune_e2e_on_instruct-60k-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/med-pretrain-364m-3epoch',\n",
       " 'llava-med/llava-vicuna-7b/llava-vicuna-7b-pretrain_cc3m_595k_1e-instruct_158k-3epoch/eval']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpts_eval_data_RAD_zs_difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llava",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}