eval.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bCTFHZEZQydn",
        "outputId": "db7d2075-0fd0-4f65-9441-77e94aadd608"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting rouge-score==0.0.4\n",
            "  Downloading rouge_score-0.0.4-py2.py3-none-any.whl.metadata (3.8 kB)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.0.4) (1.4.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.0.4) (3.8.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.0.4) (1.26.4)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.0.4) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score==0.0.4) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score==0.0.4) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score==0.0.4) (2024.5.15)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score==0.0.4) (4.66.5)\n",
            "Downloading rouge_score-0.0.4-py2.py3-none-any.whl (22 kB)\n",
            "Installing collected packages: rouge-score\n",
            "Successfully installed rouge-score-0.0.4\n"
          ]
        }
      ],
      "source": [
        "!pip install rouge-score==0.0.4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qZSQNxNSUedN",
        "outputId": "27c79de1-e759-49ec-bcf6-feb7982927cd"
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "import json\n",
        "from tqdm  import tqdm\n",
        "import nltk\n",
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "from nltk.translate.bleu_score import SmoothingFunction\n",
        "import os\n",
        "import tensorflow as tf\n",
        "from rouge_score import rouge_scorer\n",
        "# import bleurt.score\n",
        "\n",
        "nltk.download('omw-1.4')\n",
        "nltk.download('punkt')\n",
        "\n",
        "\n",
        "def calculate_vqa_score(standard_answer, generated_answer):\n",
        "    \"\"\"\n",
        "    计算标准答案和生成答案之间的VQA分数。\n",
        "\n",
        "    参数:\n",
        "    standard_answer -- 标准答案字符串\n",
        "    generated_answer -- 模型生成的答案字符串\n",
        "\n",
        "    返回:\n",
        "    score -- VQA得分（0到1之间）\n",
        "    \"\"\"\n",
        "    # 创建TfidfVectorizer对象\n",
        "    vectorizer = TfidfVectorizer()\n",
        "\n",
        "    # 将答案转化为TF-IDF矩阵\n",
        "    tfidf_matrix = vectorizer.fit_transform([standard_answer, generated_answer])\n",
        "\n",
        "    # 使用余弦相似度计算相似度\n",
        "    score = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0]\n",
        "\n",
        "    return score\n",
        "\n",
        "def calculate_bleu_score(reference_answer, candidate_answer):\n",
        "    \"\"\"\n",
        "    计算候选答案与参考答案之间的Bleu Score。\n",
        "\n",
        "    参数:\n",
        "    reference_answer -- 参考答案列表，可以包含多个参考答案\n",
        "    candidate_answer -- 模型生成的答案字符串\n",
        "\n",
        "    返回:\n",
        "    bleu_score -- Bleu Score\n",
        "    \"\"\"\n",
        "    # 预处理：将字符串转换为小写并移除两端空白字符\n",
        "    reference_answer = [ref.lower().strip().split() for ref in reference_answer]\n",
        "    candidate_answer = candidate_answer.lower().strip().split()\n",
        "\n",
        "    # 计算Bleu Score\n",
        "    smoothing = SmoothingFunction().method1  # 为了避免分母为零的情况\n",
        "    bleu_score = sentence_bleu(reference_answer, candidate_answer, smoothing_function=smoothing)\n",
        "\n",
        "    return bleu_score\n",
        "\n",
        "\n",
        "def calculate_rouge_l(standard_answer, generated_answer):\n",
        "    \"\"\"\n",
        "    计算标准答案与生成答案之间的 ROUGE-L 分数。\n",
        "\n",
        "    参数:\n",
        "    standard_answer -- 标准答案字符串\n",
        "    generated_answer -- 模型生成的答案字符串\n",
        "\n",
        "    返回:\n",
        "    scores -- 包含 ROUGE-L 分数的字典\n",
        "    \"\"\"\n",
        "    # 初始化 ROUGE 计分器\n",
        "    scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=False)\n",
        "\n",
        "    # 计算 ROUGE-L 分数\n",
        "    scores = scorer.score(standard_answer, generated_answer)\n",
        "\n",
        "    return scores\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "directory = \"data\"\n",
        "scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)\n",
        "scores = []\n",
        "for filename in os.listdir(directory):\n",
        "    n = 0\n",
        "    if filename.endswith(\".json\"):\n",
        "      filepath = os.path.join(directory, filename)\n",
        "      with open(filepath, 'r') as f:\n",
        "        data = json.load(f)\n",
        "        vqa_score = 0\n",
        "        bleu_score = 0\n",
        "        R_scores = 0\n",
        "        for item in tqdm(data):\n",
        "\n",
        "            for item2 in item[\"Q&A\"]:\n",
        "                standard_answer = item2[\"ground_truth\"]\n",
        "                generated_answer = item2[\"answer\"]\n",
        "                vqa_score += calculate_vqa_score(standard_answer, generated_answer)\n",
        "                bleu_score += calculate_bleu_score(standard_answer, generated_answer)\n",
        "                score = calculate_rouge_l(standard_answer, generated_answer)\n",
        "                scores.append(score['rougeL'])\n",
        "                n += 1\n",
        "        avg_precision = sum(score.precision for score in scores) / len(scores)\n",
        "        avg_recall = sum(score.recall for score in scores) / len(scores)\n",
        "        avg_f_measure = sum(score.fmeasure for score in scores) / len(scores)\n",
        "        average_scores = {\n",
        "        'rougeL': {\n",
        "            'precision': avg_precision,\n",
        "            'recall': avg_recall,\n",
        "            'fmeasure': avg_f_measure\n",
        "          }\n",
        "        }\n",
        "\n",
        "        fin_vqa_score = vqa_score / n\n",
        "        fin_bleu_score = bleu_score / n\n",
        "\n",
        "\n",
        "\n",
        "        print(f\"Filename: {filename}\")\n",
        "        print(f\"VQA Score: {fin_vqa_score:.4f}\")\n",
        "        print(f\"bleu Score: {fin_bleu_score:.4f}\")\n",
        "        for key, value in average_scores.items():\n",
        "          print(f\"{key}: Precision={value['precision']:.4f}, Recall={value['recall']:.4f}, F-Measure={value['fmeasure']:.4f}\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}