geobench/baseline_results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import geobench\n",
    "if geobench.config._GEO_BENCH_DIR_ENV is None:\n",
    "    print(\"Note: GEO_BENCH_DIR is not set. If this is not intended, this might be related on how jupyter loads environment variables.\")\n",
    "    print(f\"Using GEO_BENCH_DIR={geobench.GEO_BENCH_DIR}\")\n",
    "\n",
    "import plot_tools\n",
    "from pathlib import Path\n",
    "import seaborn as sns\n",
    "\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_path = Path(os.path.abspath('')).parent / \"baseline_classification_results.csv\"\n",
    "df_all = pd.read_csv(csv_path)\n",
    "df_1x = plot_tools.extract_1x_data(df_all)\n",
    "\n",
    "model_order = \"ResNet18-Rnd,ResNet18-timm,ResNet18-MoCo-S2,ResNet50-MillionAID,ResNet50-MoCo-S2,ResNet50-timm,ConvNeXt-B-timm,ViT-T-timm,ViT-S-timm,SwinV2-T-timm\".split(\",\")\n",
    "model_colors = dict( zip(model_order, sns.color_palette(\"colorblind\")[:len(model_order)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Raw seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_tools.plot_per_dataset(df_1x, model_order, model_colors=model_colors, metric=\"test metric\", sharey=False, inner=\"points\", fig_size=(14,3), n_legend_rows=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bootstrap IQM and aggregate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "plot_tools.normalize_bootstrap_and_plot(df_1x, metric=\"test metric\",benchmark_name=\"classification_v1.0\", model_order=model_order, model_colors=model_colors, fig_size=(12,2.3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tabular Resulst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_tools.avergage_seeds(df_1x)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Segmentation\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "csv_path = Path(os.path.abspath('')).parent / \"baseline_segmentation_results.csv\"\n",
    "df_all = pd.read_csv(csv_path)\n",
    "df_1x = plot_tools.extract_1x_data(df_all)\n",
    "model_order = 'ResNet18-U-Net-timm,ResNet50-U-Net-timm,ResNet101-U-Net-timm,ResNet18 DeepLabV3-timm,ResNet50 DeepLabV3-timm,ResNet101 DeepLabV3-timm'.split(',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Raw seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_tools.plot_per_dataset(df_1x, model_order, metric=\"test metric\", sharey=False, inner=\"points\", fig_size=(14,3), n_legend_rows=2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bootstrap IQM and aggregate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_tools.normalize_bootstrap_and_plot(df_1x, metric=\"test metric\",benchmark_name=\"segmentation_v1.0\", model_order=model_order, fig_size=(12,2.3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tabular results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_tools.avergage_seeds(df_1x)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}