evaluation/Evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "gpuClass": "premium",
      "mount_file_id": "1aG2QvDNLv5Xwwqi6NymBdJXcIPY4CSgj",
      "authorship_tag": "ABX9TyPks/TfJiUlIOM4KgFPc1Dt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "premium",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ymoslem/Adaptive-MT-LLM/blob/main/evaluation/Evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load files"
      ],
      "metadata": {
        "id": "xTcWVCFXjDGM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir(\"/content/drive/MyDrive/data\")\n",
        "\n",
        "\n",
        "# ✳️ Change file names\n",
        "source_file = \"tico-19-enes-dedup.en\" #@param [\"tico-19-enar-dedup.en\", \"tico-19-enes-dedup.en\", \"tico-19-enfr-dedup.en\", \"tico-19-enrw-dedup.en\", \"tico-19-enzh-dedup.en\"] {allow-input: true}\n",
        "target_file = \"tico-19-enes-dedup.es\" #@param [\"tico-19-enar-dedup.ar\", \"tico-19-enes-dedup.es\", \"tico-19-enfr-dedup.fr\", \"tico-19-enrw-dedup.rw\", \"tico-19-enzh-dedup.zh\"] {allow-input: true}\n",
        "translation_file = \"tico-enes.gpt4-2-shot-similar.es\" #@param {type:\"string\"}\n",
        "\n",
        "# Load translations\n",
        "directory = \"MT\" #@param [\"MT\", \"outputs\"]\n",
        "file_path = os.path.join(directory, translation_file)\n",
        "\n",
        "with open(file_path, \"r\") as translation:\n",
        "  translations = [sent.strip() for sent in translation.readlines()]\n",
        "  print(len(translations))\n",
        "  print(translations[0])\n",
        "\n",
        "# Load source\n",
        "with open(source_file, \"r\") as source:\n",
        "  source_sentences = [sent.strip() for sent in source.readlines()]\n",
        "  print(len(source_sentences))\n",
        "  print(source_sentences[0])\n",
        "\n",
        "# Load target (references)\n",
        "with open(target_file, \"r\") as reference:\n",
        "  target_sentences = [sent.strip() for sent in reference.readlines()]\n",
        "  print(len(target_sentences))\n",
        "  print(target_sentences[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vLhlhV_G3xa8",
        "outputId": "0ff76ec2-2294-4963-fa65-da9e59112e38"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3070\n",
            "Priorice soluciones para abordar desafíos psicosociales antes de enseñar: movilice las herramientas disponibles para conectar escuelas, padres, docentes y estudiantes entre sí.\n",
            "3070\n",
            "Prioritize solutions to address psychosocial challenges before teaching: Mobilize available tools to connect schools, parents, teachers, and students with each other.\n",
            "3070\n",
            "Dé prioridad a las soluciones que abordan problemáticas psicosociales antes de dar clases: movilice las herramientas disponibles para conectar a las escuelas, los padres, los docentes y los estudiantes entre sí.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check if there are missing translations\n",
        "count = 0\n",
        "for idx, line in enumerate(translations):\n",
        "  if len(line.strip()) == 0:\n",
        "    count += 1\n",
        "    print(idx)\n",
        "    print(source_sentences[idx].strip())\n",
        "print(\"Missing translations:\", count)"
      ],
      "metadata": {
        "id": "HABK8IcXbKn4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27297d5a-a3a9-43e2-c4f9-d616293eee86"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Missing translations: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation"
      ],
      "metadata": {
        "id": "AzALzfsu7vl4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Calculate BLEU, CHRF and TER"
      ],
      "metadata": {
        "id": "8eRtpHae9-mt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install sacrebleu sentencepiece -q"
      ],
      "metadata": {
        "id": "l_NWEb9XZ-2b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "949b3dc5-1eaf-4614-893d-f76f71e01302"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.9/118.9 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m46.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sacrebleu\n",
        "\n",
        "references = target_sentences\n",
        "translations = translations\n",
        "\n",
        "\n",
        "# Calculate BLEU\n",
        "bleu = sacrebleu.corpus_bleu(translations, [references])  # for spBLEU: tokenize='flores200'\n",
        "print(\"BLEU:\", round(bleu.score, 2))\n",
        "\n",
        "# Calculate CHRF\n",
        "chrf = sacrebleu.corpus_chrf(translations, [references])\n",
        "print(\"CHRF:\", round(chrf.score, 2))\n",
        "\n",
        "# Calculate TER\n",
        "metric = sacrebleu.metrics.TER()\n",
        "ter = metric.corpus_score(translations, [references])\n",
        "print(\"TER:\", round(ter.score, 2))"
      ],
      "metadata": {
        "id": "RhJmD1F47xVi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fef93c03-cfd5-4bfe-ae5f-af153fb087d1"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU: 56.08\n",
            "CHRF: 76.51\n",
            "TER: 31.77\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Calculate COMET"
      ],
      "metadata": {
        "id": "0BOmakuL-HjO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['TRANSFORMERS_CACHE'] = \"/content/drive/MyDrive/models/\"\n",
        "\n",
        "!pip3 install unbabel-comet -q"
      ],
      "metadata": {
        "id": "5gq_5b5wyc5C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be293290-545f-47d4-c8c5-d1350bbc0ca5"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/81.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.6/81.6 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.4/101.4 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m829.5/829.5 kB\u001b[0m \u001b[31m57.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m409.8/409.8 kB\u001b[0m \u001b[31m43.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m110.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m98.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m115.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m118.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m105.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m109.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m105.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from comet import download_model, load_from_checkpoint\n",
        "import pandas as pd\n",
        "\n",
        "references = target_sentences\n",
        "translations = translations\n",
        "\n",
        "# Calculate COMET\n",
        "df = pd.DataFrame({\"src\":source_sentences, \"mt\":translations, \"ref\":references})\n",
        "data = df.to_dict('records')\n",
        "#model_path = download_model(\"wmt20-comet-da\")  # to download the model if you did not yet\n",
        "model_path = \"/content/drive/MyDrive/models/wmt20-comet-da/checkpoints/model.ckpt\"\n",
        "model = load_from_checkpoint(model_path)\n",
        "\n",
        "seg_scores, sys_score = model.predict(data, batch_size=128, gpus=1).values()\n",
        "print(\"COMET:\", round(sys_score*100, 2))"
      ],
      "metadata": {
        "id": "PhLOXCyMrOxD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b97dd7e7-7c66-414d-9e08-f74fe45334ed"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:pytorch_lightning.utilities.migration.utils:Lightning automatically upgraded your loaded checkpoint from v1.3.5 to v1.9.5. To apply the upgrade to your files permanently, run `python -m pytorch_lightning.utilities.upgrade_checkpoint --file ../models/wmt20-comet-da/checkpoints/model.ckpt`\n",
            "INFO:pytorch_lightning.utilities.rank_zero:GPU available: True (cuda), used: True\n",
            "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
            "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
            "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n",
            "INFO:pytorch_lightning.accelerators.cuda:LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "Predicting DataLoader 0: 100%|██████████| 24/24 [00:33<00:00,  1.39s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "COMET: 91.86\n"
          ]
        }
      ]
    }
  ]
}