{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from openai import OpenAI\n", "import json\n", "import os\n", "from tqdm import tqdm\n", "import pandas as pd\n", "import numpy as np\n", "from collections import Counter\n", "import time\n", "import pathlib\n", "client = OpenAI(\n", " # This is the default and can be omitted\n", " api_key=os.environ.get(\"OPENAI_API_KEY\"),\n", ")" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "gpt_model = \"gpt-4-0613\"\n", "\n", "\n", "prompt = \"\"\"Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. in the ground truth means it is totally right only when all elements in the ground truth are present in the prediction, and means it is totally right when any one element in the ground truth is present in the prediction. The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Just complete the last space of the correctness score.\n", "\n", "Question | Ground truth | Prediction | Correctness\n", "--- | --- | --- | ---\n", "What is x in the equation? | -1 -5 | x = 3 | 0.0\n", "What is x in the equation? | -1 -5 | x = -1 | 0.5\n", "What is x in the equation? | -1 -5 | x = -5 | 0.5\n", "What is x in the equation? | -1 -5 | x = -5 or 5 | 0.5\n", "What is x in the equation? | -1 -5 | x = -1 or x = -5 | 1.0\n", "Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme talks about Iceland and Greenland. It's pointing out that despite their names, Iceland is not very icy and Greenland isn't very green. | 0.4\n", "Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. The text 'This is why I have trust issues' is a playful way to suggest that these contradictions can lead to distrust or confusion. The humor in this meme is derived from the unexpected contrast between the names of the countries and their actual physical characteristics. | 1.0\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# load metadata\n", "# Download mm-vet.zip and `unzip mm-vet.zip` and change the path below\n", "mmvet_path = \"/path/to/mm-vet\"\n", "use_sub_set = False\n", "decimal_places = 1 # number of decimal places to round to\n", "\n", "\n", "if use_sub_set:\n", " bard_set_file = os.path.join(mmvet_path, \"bard_set.json\")\n", " with open(bard_set_file, 'r') as f:\n", " sub_set = json.load(f)\n", " sub_set_name = 'bardset'\n", " sub_set_name = sub_set_name + '_'\n", "else:\n", " sub_set = None\n", " sub_set_name = ''\n", "\n", "mmvet_metadata = os.path.join(mmvet_path, \"mm-vet.json\")\n", "with open(mmvet_metadata, 'r') as f:\n", " data = json.load(f)\n", "\n", "\n", "counter = Counter()\n", "cap_set_list = []\n", "cap_set_counter = []\n", "len_data = 0\n", "for id, value in data.items():\n", " if sub_set is not None and id not in sub_set:\n", " continue\n", " question = value[\"question\"]\n", " answer = value[\"answer\"]\n", " cap = value[\"capability\"]\n", " cap = set(cap)\n", " counter.update(cap)\n", " if cap not in cap_set_list:\n", " cap_set_list.append(cap)\n", " cap_set_counter.append(1)\n", " else:\n", " cap_set_counter[cap_set_list.index(cap)] += 1\n", " \n", " len_data += 1\n", "\n", "sorted_list = counter.most_common()\n", "columns = [k for k, v in sorted_list]\n", "columns.append(\"total\")\n", "columns.append(\"std\")\n", "columns.append('runs')\n", "df = pd.DataFrame(columns=columns)\n", "\n", "\n", "cap_set_sorted_indices = np.argsort(-np.array(cap_set_counter))\n", "new_cap_set_list = []\n", "new_cap_set_counter = []\n", "for index in cap_set_sorted_indices:\n", " new_cap_set_list.append(cap_set_list[index])\n", " new_cap_set_counter.append(cap_set_counter[index])\n", "\n", "cap_set_list = new_cap_set_list\n", "cap_set_counter = new_cap_set_counter\n", "cap_set_names = [\"_\".join(list(cap_set)) for cap_set in cap_set_list]\n", "\n", "columns2 = cap_set_names\n", "columns2.append(\"total\")\n", "columns2.append(\"std\")\n", "columns2.append('runs')\n", "df2 = pd.DataFrame(columns=columns2)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "result_file = \"results/llava_llama2_13b_chat.json\" # change your model result_file\n", "result_path = \"results\" # path to save grading results\n", "num_run = 1 # we set it as 5 in the paper\n", "\n", "if os.path.exists(result_file) is False:\n", " raise ValueError(\"Result file does not exist\")\n", "if not result_file.endswith(('.json', '.JSON')):\n", " raise ValueError(\"Result file should be a json file\")\n", "model = pathlib.Path(result_file).stem\n", "# grade results for each sample to svae\n", "grade_file = f'{model}_{gpt_model}-grade-{num_run}runs.json'\n", "grade_file = os.path.join(result_path, grade_file)\n", "\n", "# score results regarding capabilities/capability integration to save\n", "cap_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-score-{num_run}runs.csv'\n", "cap_score_file = os.path.join(result_path, cap_score_file)\n", "cap_int_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-int-score-{num_run}runs.csv'\n", "cap_int_score_file = os.path.join(result_path, cap_int_score_file)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "eval run 0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/218 [00:00 0:\n", " for k, v in grade_results.items():\n", " if len(v['score']) < num_run:\n", " need_more_runs = True\n", " break\n", " return need_more_runs or len(grade_results) < len_data\n", "\n", "\n", "while need_more_runs():\n", " for j in range(num_run):\n", " print(f'eval run {j}')\n", " for id, line in tqdm(data.items()):\n", " if sub_set is not None and id not in sub_set:\n", " continue\n", " if id in grade_results and len(grade_results[id]['score']) >= (j + 1):\n", " continue\n", "\n", " model_pred = results[id]\n", " \n", " question = prompt + '\\n' + ' | '.join([line['question'], line['answer'].replace(\"\", \" \").replace(\"\", \" \"), model_pred, \"\"])\n", " messages = [\n", " {\"role\": \"user\", \"content\": question},\n", " ]\n", "\n", " if id not in grade_results:\n", " sample_grade = {'model': [], 'content': [], 'score': []}\n", " else:\n", " sample_grade = grade_results[id]\n", "\n", " \n", " grade_sample_run_complete = False\n", " temperature = 0.0\n", "\n", " while not grade_sample_run_complete:\n", " try:\n", " response = client.chat.completions.create(\n", " model=gpt_model,\n", " max_tokens=3,\n", " temperature=temperature,\n", " messages=messages)\n", " content = response.choices[0].message.content\n", " flag = True\n", " try_time = 1\n", " while flag:\n", " try:\n", " content = content.split(' ')[0].strip()\n", " score = float(content)\n", " if score > 1.0 or score < 0.0:\n", " assert False\n", " flag = False\n", " except:\n", " question = prompt + '\\n' + ' | '.join([line['question'], line['answer'].replace(\"\", \" \").replace(\"\", \" \"), model_pred, \"\"]) + \"\\nPredict the correctness of the answer (digit): \"\n", " messages = [\n", " {\"role\": \"user\", \"content\": question},\n", " ]\n", " response = client.chat.completions.create(\n", " model=gpt_model,\n", " max_tokens=3,\n", " temperature=temperature,\n", " messages=messages)\n", " content = response.choices[0].message.content\n", " try_time += 1\n", " temperature += 0.5\n", " print(f\"{id} try {try_time} times\")\n", " print(content)\n", " if try_time > 5:\n", " score = 0.0\n", " flag = False\n", " grade_sample_run_complete = True\n", " except:\n", " # gpt4 may have token rate limit\n", " print(\"sleep 30s\")\n", " time.sleep(30)\n", "\n", " if len(sample_grade['model']) >= j + 1:\n", " sample_grade['model'][j] = response.model\n", " sample_grade['content'][j] = content\n", " sample_grade['score'][j] = score\n", " else:\n", " sample_grade['model'].append(response.model)\n", " sample_grade['content'].append(content)\n", " sample_grade['score'].append(score)\n", " grade_results[id] = sample_grade\n", "\n", " with open(grade_file, 'w') as f:\n", " json.dump(grade_results, f, indent=4)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "assert not need_more_runs()\n", "cap_socres = {k: [0.0]*num_run for k in columns[:-2]}\n", "counter['total'] = len_data\n", "\n", "cap_socres2 = {k: [0.0]*num_run for k in columns2[:-2]}\n", "counter2 = {columns2[i]:cap_set_counter[i] for i in range(len(cap_set_counter))}\n", "counter2['total'] = len_data\n", "\n", "for k, v in grade_results.items():\n", " if sub_set is not None and k not in sub_set:\n", " continue\n", " for i in range(num_run):\n", " score = v['score'][i]\n", " caps = set(data[k]['capability'])\n", " for c in caps:\n", " cap_socres[c][i] += score\n", " \n", " cap_socres['total'][i] += score\n", "\n", " index = cap_set_list.index(caps)\n", " cap_socres2[cap_set_names[index]][i] += score\n", " cap_socres2['total'][i] += score\n", "\n", "for k, v in cap_socres.items():\n", " cap_socres[k] = np.array(v) / counter[k] *100\n", "\n", "\n", "std = round(cap_socres['total'].std(), decimal_places)\n", "total_copy = cap_socres['total'].copy()\n", "runs = str(list(np.round(total_copy, decimal_places)))\n", "\n", "for k, v in cap_socres.items():\n", " cap_socres[k] = round(v.mean(), decimal_places)\n", "\n", "cap_socres['std'] = std\n", "cap_socres['runs'] = runs\n", "df.loc[model] = cap_socres\n", "\n", "\n", "for k, v in cap_socres2.items():\n", " cap_socres2[k] = round(np.mean(np.array(v) / counter2[k] *100), decimal_places)\n", "cap_socres2['std'] = std\n", "cap_socres2['runs'] = runs\n", "df2.loc[model] = cap_socres2\n", "\n", "df.to_csv(cap_score_file)\n", "df2.to_csv(cap_int_score_file)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
recocrknowgenspatmathtotalstdruns
llava_llama2_13b_chat39.723.227.130.430.87.733.30.0[33.3]
\n", "
" ], "text/plain": [ " rec ocr know gen spat math total std runs\n", "llava_llama2_13b_chat 39.7 23.2 27.1 30.4 30.8 7.7 33.3 0.0 [33.3]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# when use subset, please note the column order is different from the full set\n", "# because it ranks by numbers of capabilties/capability integrations\n", "df" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
rec_know_genrecocr_spatocr_spat_mathrec_spatocrocr_mathrec_knowocr_rec_know_genocr_rec_spat_genocr_rec_spatocr_recocr_spat_knowrec_spat_knowocr_spat_genocr_rec_spat_mathtotalstdruns
llava_llama2_13b_chat30.559.523.514.358.331.70.027.85.060.028.650.033.30.010.00.033.30.0[33.3]
\n", "
" ], "text/plain": [ " rec_know_gen rec ocr_spat ocr_spat_math rec_spat \\\n", "llava_llama2_13b_chat 30.5 59.5 23.5 14.3 58.3 \n", "\n", " ocr ocr_math rec_know ocr_rec_know_gen \\\n", "llava_llama2_13b_chat 31.7 0.0 27.8 5.0 \n", "\n", " ocr_rec_spat_gen ocr_rec_spat ocr_rec ocr_spat_know \\\n", "llava_llama2_13b_chat 60.0 28.6 50.0 33.3 \n", "\n", " rec_spat_know ocr_spat_gen ocr_rec_spat_math total \\\n", "llava_llama2_13b_chat 0.0 10.0 0.0 33.3 \n", "\n", " std runs \n", "llava_llama2_13b_chat 0.0 [33.3] " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df2" ] } ], "metadata": { "kernelspec": { "display_name": "base", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }