{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "234f5da7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scienceplots  # noqa: F401\n",
    "import seaborn as sns\n",
    "import seaborn.objects as so\n",
    "import shap.plots\n",
    "from rich import pretty, traceback\n",
    "from scipy.stats import f_oneway, shapiro\n",
    "\n",
    "from entropice.utils.paths import RESULTS_DIR\n",
    "from entropice.utils.training import Training\n",
    "\n",
    "pretty.install()\n",
    "traceback.install()\n",
    "\n",
    "sns.set_theme(context=\"paper\", style=\"whitegrid\", palette=\"tab10\", font_scale=1.5)\n",
    "plt.style.use([\"science\", \"grid\", \"no-latex\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "073e4e89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 81 runs for experiment_name='feature_importance_era5-shoulder_arcticdem-v2'\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Found 36 runs for experiment_name=\"feature_importance_era5-shoulder_arcticdem-v2_selection='univariate'\"\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Warning: Post-merge filters are not checked for validity! Please ensure that they are valid columns in the final merged dataset.\n",
      "Found 36 runs for experiment_name=\"feature_importance_era5-shoulder_arcticdem-v2_selection='cluster'\"\n",
      "Found 81 runs for experiment_name='feature_importance_alphaearth'\n",
      "Found 234 (130 binary, 104 density) runs in total across 4 experiments.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>dataset</th>\n",
       "      <th>grid</th>\n",
       "      <th>level</th>\n",
       "      <th>members</th>\n",
       "      <th>method</th>\n",
       "      <th>task</th>\n",
       "      <th>target</th>\n",
       "      <th>model_type</th>\n",
       "      <th>run</th>\n",
       "      <th>...</th>\n",
       "      <th>train_pearsonr</th>\n",
       "      <th>train_median_absolute_error</th>\n",
       "      <th>test_root_mean_squared_error</th>\n",
       "      <th>test_pearsonr</th>\n",
       "      <th>test_median_absolute_error</th>\n",
       "      <th>complete_root_mean_squared_error</th>\n",
       "      <th>complete_pearsonr</th>\n",
       "      <th>complete_median_absolute_error</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>experiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-3['ArcticDEM', 'ERA5-shoulder']</td>\n",
       "      <td>H3</td>\n",
       "      <td>3</td>\n",
       "      <td>[ArcticDEM, ERA5-shoulder]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-9['ArcticDEM', 'ERA5-shoulder']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>9</td>\n",
       "      <td>[ArcticDEM, ERA5-shoulder]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-5['ArcticDEM', 'ERA5-shoulder']</td>\n",
       "      <td>H3</td>\n",
       "      <td>5</td>\n",
       "      <td>[ArcticDEM, ERA5-shoulder]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-3['ArcticDEM', 'ERA5-shoulder']</td>\n",
       "      <td>H3</td>\n",
       "      <td>3</td>\n",
       "      <td>[ArcticDEM, ERA5-shoulder]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-6['ArcticDEM', 'ERA5-shoulder']</td>\n",
       "      <td>H3</td>\n",
       "      <td>6</td>\n",
       "      <td>[ArcticDEM, ERA5-shoulder]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>density</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-6['AlphaEarth']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>6</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>density</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.690132</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>0.587129</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000225</td>\n",
       "      <td>0.668917</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>AlphaEarth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-10['AlphaEarth']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>10</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>AlphaEarth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-6['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>6</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>AlphaEarth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-5['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>5</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>AlphaEarth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-6['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>6</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>density</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>kNN</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>AlphaEarth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>234 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                                  path  \\\n",
       "\u001b[1;36m0\u001b[0m    \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m    \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m    \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m    \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m    \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "..                                                 \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m229\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m230\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m231\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m232\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m233\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\n",
       "                                     dataset     grid  level  \\\n",
       "\u001b[1;36m0\u001b[0m        hex-\u001b[1;36m3\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'ArcticDEM'\u001b[0m, \u001b[32m'ERA5-shoulder'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m    healpix-\u001b[1;36m9\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'ArcticDEM'\u001b[0m, \u001b[32m'ERA5-shoulder'\u001b[0m\u001b[1m]\u001b[0m  HEALPix      \u001b[1;36m9\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m        hex-\u001b[1;36m5\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'ArcticDEM'\u001b[0m, \u001b[32m'ERA5-shoulder'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m        hex-\u001b[1;36m3\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'ArcticDEM'\u001b[0m, \u001b[32m'ERA5-shoulder'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m        hex-\u001b[1;36m6\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'ArcticDEM'\u001b[0m, \u001b[32m'ERA5-shoulder'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m6\u001b[0m   \n",
       "..                                       \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m229\u001b[0m                  healpix-\u001b[1;36m6\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m  HEALPix      \u001b[1;36m6\u001b[0m   \n",
       "\u001b[1;36m230\u001b[0m                 healpix-\u001b[1;36m10\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m  HEALPix     \u001b[1;36m10\u001b[0m   \n",
       "\u001b[1;36m231\u001b[0m                      hex-\u001b[1;36m6\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m6\u001b[0m   \n",
       "\u001b[1;36m232\u001b[0m                      hex-\u001b[1;36m5\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m233\u001b[0m                      hex-\u001b[1;36m6\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m6\u001b[0m   \n",
       "\n",
       "                        members  method     task    target     model_type  \\\n",
       "\u001b[1;36m0\u001b[0m    \u001b[1m[\u001b[0mArcticDEM, ERA5-shoulder\u001b[1m]\u001b[0m   HPOCV   binary  darts_v1           eSPA   \n",
       "\u001b[1;36m1\u001b[0m    \u001b[1m[\u001b[0mArcticDEM, ERA5-shoulder\u001b[1m]\u001b[0m   HPOCV   binary  darts_v1           eSPA   \n",
       "\u001b[1;36m2\u001b[0m    \u001b[1m[\u001b[0mArcticDEM, ERA5-shoulder\u001b[1m]\u001b[0m   HPOCV   binary  darts_v1        XGBoost   \n",
       "\u001b[1;36m3\u001b[0m    \u001b[1m[\u001b[0mArcticDEM, ERA5-shoulder\u001b[1m]\u001b[0m   HPOCV   binary  darts_v1  Random Forest   \n",
       "\u001b[1;36m4\u001b[0m    \u001b[1m[\u001b[0mArcticDEM, ERA5-shoulder\u001b[1m]\u001b[0m   HPOCV  density  darts_v1  Random Forest   \n",
       "..                          \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m229\u001b[0m                \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML  density  darts_v1      AutoGluon   \n",
       "\u001b[1;36m230\u001b[0m                \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML   binary  darts_v1      AutoGluon   \n",
       "\u001b[1;36m231\u001b[0m                \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML   binary  darts_v1      AutoGluon   \n",
       "\u001b[1;36m232\u001b[0m                \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m   HPOCV   binary  darts_v1  Random Forest   \n",
       "\u001b[1;36m233\u001b[0m                \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m   HPOCV  density  darts_v1            kNN   \n",
       "\n",
       "                                                   run  \u001b[33m...\u001b[0m  train_pearsonr  \\\n",
       "\u001b[1;36m0\u001b[0m    \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\u001b[1;36m1\u001b[0m    \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\u001b[1;36m2\u001b[0m    \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\u001b[1;36m3\u001b[0m    \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\u001b[1;36m4\u001b[0m    \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "..                                                 \u001b[33m...\u001b[0m  \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m229\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m        \u001b[1;36m0.690132\u001b[0m   \n",
       "\u001b[1;36m230\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\u001b[1;36m231\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\u001b[1;36m232\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\u001b[1;36m233\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m             NaN   \n",
       "\n",
       "     train_median_absolute_error  test_root_mean_squared_error  test_pearsonr  \\\n",
       "\u001b[1;36m0\u001b[0m                            NaN                           NaN            NaN   \n",
       "\u001b[1;36m1\u001b[0m                            NaN                           NaN            NaN   \n",
       "\u001b[1;36m2\u001b[0m                            NaN                           NaN            NaN   \n",
       "\u001b[1;36m3\u001b[0m                            NaN                           NaN            NaN   \n",
       "\u001b[1;36m4\u001b[0m                            NaN                           NaN            NaN   \n",
       "..                           \u001b[33m...\u001b[0m                           \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m229\u001b[0m                     \u001b[1;36m0.000008\u001b[0m                      \u001b[1;36m0.000131\u001b[0m       \u001b[1;36m0.587129\u001b[0m   \n",
       "\u001b[1;36m230\u001b[0m                          NaN                           NaN            NaN   \n",
       "\u001b[1;36m231\u001b[0m                          NaN                           NaN            NaN   \n",
       "\u001b[1;36m232\u001b[0m                          NaN                           NaN            NaN   \n",
       "\u001b[1;36m233\u001b[0m                          NaN                           NaN            NaN   \n",
       "\n",
       "     test_median_absolute_error  complete_root_mean_squared_error  \\\n",
       "\u001b[1;36m0\u001b[0m                           NaN                               NaN   \n",
       "\u001b[1;36m1\u001b[0m                           NaN                               NaN   \n",
       "\u001b[1;36m2\u001b[0m                           NaN                               NaN   \n",
       "\u001b[1;36m3\u001b[0m                           NaN                               NaN   \n",
       "\u001b[1;36m4\u001b[0m                           NaN                               NaN   \n",
       "..                          \u001b[33m...\u001b[0m                               \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m229\u001b[0m                    \u001b[1;36m0.000009\u001b[0m                          \u001b[1;36m0.000225\u001b[0m   \n",
       "\u001b[1;36m230\u001b[0m                         NaN                               NaN   \n",
       "\u001b[1;36m231\u001b[0m                         NaN                               NaN   \n",
       "\u001b[1;36m232\u001b[0m                         NaN                               NaN   \n",
       "\u001b[1;36m233\u001b[0m                         NaN                               NaN   \n",
       "\n",
       "     complete_pearsonr  complete_median_absolute_error        grid_level  \\\n",
       "\u001b[1;36m0\u001b[0m                  NaN                             NaN        H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                  NaN                             NaN   HEALPix Level \u001b[1;36m9\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m                  NaN                             NaN        H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                  NaN                             NaN        H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                  NaN                             NaN        H3 Level \u001b[1;36m6\u001b[0m   \n",
       "..                 \u001b[33m...\u001b[0m                             \u001b[33m...\u001b[0m               \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m229\u001b[0m           \u001b[1;36m0.668917\u001b[0m                        \u001b[1;36m0.000008\u001b[0m   HEALPix Level \u001b[1;36m6\u001b[0m   \n",
       "\u001b[1;36m230\u001b[0m                NaN                             NaN  HEALPix Level \u001b[1;36m10\u001b[0m   \n",
       "\u001b[1;36m231\u001b[0m                NaN                             NaN        H3 Level \u001b[1;36m6\u001b[0m   \n",
       "\u001b[1;36m232\u001b[0m                NaN                             NaN        H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m233\u001b[0m                NaN                             NaN        H3 Level \u001b[1;36m6\u001b[0m   \n",
       "\n",
       "                experiment  \n",
       "\u001b[1;36m0\u001b[0m    ERA5-Land + ArcticDEM  \n",
       "\u001b[1;36m1\u001b[0m    ERA5-Land + ArcticDEM  \n",
       "\u001b[1;36m2\u001b[0m    ERA5-Land + ArcticDEM  \n",
       "\u001b[1;36m3\u001b[0m    ERA5-Land + ArcticDEM  \n",
       "\u001b[1;36m4\u001b[0m    ERA5-Land + ArcticDEM  \n",
       "..                     \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m229\u001b[0m             AlphaEarth  \n",
       "\u001b[1;36m230\u001b[0m             AlphaEarth  \n",
       "\u001b[1;36m231\u001b[0m             AlphaEarth  \n",
       "\u001b[1;36m232\u001b[0m             AlphaEarth  \n",
       "\u001b[1;36m233\u001b[0m             AlphaEarth  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m234\u001b[0m rows x \u001b[1;36m102\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# experiment_name = \"feature_importance_era5-shoulder_arcticdem-v2\"\n",
    "# experiment_name = \"feature_importance_era5-shoulder_arcticdem-v2_selection='univariate'\"\n",
    "# experiment_name = \"feature_importance_era5-shoulder_arcticdem-v2_selection='cluster'\"\n",
    "# experiment_name = \"feature_importance_alphaearth\"\n",
    "experiment_names = [\n",
    "    \"feature_importance_era5-shoulder_arcticdem-v2\",\n",
    "    \"feature_importance_era5-shoulder_arcticdem-v2_selection='univariate'\",\n",
    "    \"feature_importance_era5-shoulder_arcticdem-v2_selection='cluster'\",\n",
    "    \"feature_importance_alphaearth\",\n",
    "]\n",
    "\n",
    "all_experiments = []\n",
    "for experiment_name in experiment_names:\n",
    "    experiment_dir = RESULTS_DIR / experiment_name\n",
    "\n",
    "    all_runs = pd.DataFrame(\n",
    "        [\n",
    "            Training.load(f).to_record()\n",
    "            for f in experiment_dir.iterdir()\n",
    "            if f.is_dir() and f.name.startswith(\"permafrost_\")\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    # Rename the models: \"rf\" to \"Random Forest\", \"xgboost\" to \"XGBoost\", \"knn\" to \"kNN\",\n",
    "    # \"espa\" to \"eSPA\" and \"autogluon\" to \"AutoGluon for better readability\n",
    "    all_runs[\"model_type\"] = (\n",
    "        all_runs[\"model_type\"]\n",
    "        .str.replace(\"rf\", \"Random Forest\", regex=False)\n",
    "        .str.replace(\"xgboost\", \"XGBoost\", regex=False)\n",
    "        .str.replace(\"knn\", \"kNN\", regex=False)\n",
    "        .str.replace(\"espa\", \"eSPA\", regex=False)\n",
    "        .str.replace(\"autogluon\", \"AutoGluon\", regex=False)\n",
    "    )\n",
    "\n",
    "    # Rename hex to H3 and healpix to HEALPix for better readability\n",
    "    all_runs[\"grid\"] = (\n",
    "        all_runs[\"grid\"].str.replace(\"hex\", \"H3\", regex=False).str.replace(\"healpix\", \"HEALPix\", regex=False)\n",
    "    )\n",
    "\n",
    "    # Create a combined \"grid_level\" column for easier plotting\n",
    "    grid_level = all_runs[\"grid\"].str.cat(all_runs[\"level\"].astype(str), sep=\" Level \")\n",
    "    # Make it categorical with a specific order\n",
    "    grid_level_cat = pd.CategoricalDtype(\n",
    "        categories=[\n",
    "            \"H3 Level 3\",\n",
    "            \"HEALPix Level 6\",\n",
    "            \"HEALPix Level 7\",\n",
    "            \"H3 Level 4\",\n",
    "            \"HEALPix Level 8\",\n",
    "            \"H3 Level 5\",\n",
    "            \"HEALPix Level 9\",\n",
    "            \"HEALPix Level 10\",\n",
    "            \"H3 Level 6\",\n",
    "        ],\n",
    "        ordered=True,\n",
    "    )\n",
    "    grid_level = grid_level.astype(grid_level_cat)\n",
    "    all_runs[\"grid_level\"] = grid_level\n",
    "\n",
    "    print(f\"Found {len(all_runs)} runs for {experiment_name=}\")\n",
    "    all_runs[\"experiment\"] = experiment_name\n",
    "    all_experiments.append(all_runs)\n",
    "all_experiments = pd.concat(all_experiments, ignore_index=True)\n",
    "\n",
    "# Rename Experiments\n",
    "experiment_rename_map = {\n",
    "    \"feature_importance_era5-shoulder_arcticdem-v2\": \"ERA5-Land + ArcticDEM\",\n",
    "    \"feature_importance_era5-shoulder_arcticdem-v2_selection='univariate'\": \"ERA5-Land + ArcticDEM (Univariate Selection)\",\n",
    "    \"feature_importance_era5-shoulder_arcticdem-v2_selection='cluster'\": \"ERA5-Land + ArcticDEM (Cluster Selection)\",\n",
    "    \"feature_importance_alphaearth\": \"AlphaEarth\",\n",
    "}\n",
    "all_experiments[\"experiment\"] = all_experiments[\"experiment\"].map(experiment_rename_map)\n",
    "\n",
    "binary_runs = all_experiments[all_experiments[\"task\"] == \"binary\"]\n",
    "density_runs = all_experiments[all_experiments[\"task\"] == \"density\"]\n",
    "# Drop all columns that are all NaN in either binary_runs or density_runs\n",
    "binary_runs = binary_runs.dropna(axis=1, how=\"all\")\n",
    "density_runs = density_runs.dropna(axis=1, how=\"all\")\n",
    "print(\n",
    "    f\"Found {len(all_experiments)} ({len(binary_runs)} binary, {len(density_runs)} density) runs in total\"\n",
    "    f\" across {len(experiment_names)} experiments.\"\n",
    ")\n",
    "\n",
    "all_experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cbbc1c8",
   "metadata": {},
   "source": [
    "## Model performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4dcec91f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>experiment</th>\n",
       "      <th>score</th>\n",
       "      <th>split</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.905759</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.762885</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.959493</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.975871</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.540843</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>775</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.501394</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>776</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.824979</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>777</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.515731</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>778</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.510555</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>779</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.903151</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>780 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "           grid_level     model_type             experiment     score  split  \\\n",
       "\u001b[1;36m0\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m           eSPA  ERA5-Land + ArcticDEM  \u001b[1;36m0.905759\u001b[0m  train   \n",
       "\u001b[1;36m1\u001b[0m     HEALPix Level \u001b[1;36m9\u001b[0m           eSPA  ERA5-Land + ArcticDEM  \u001b[1;36m0.762885\u001b[0m  train   \n",
       "\u001b[1;36m2\u001b[0m          H3 Level \u001b[1;36m5\u001b[0m        XGBoost  ERA5-Land + ArcticDEM  \u001b[1;36m0.959493\u001b[0m  train   \n",
       "\u001b[1;36m3\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m  Random Forest  ERA5-Land + ArcticDEM  \u001b[1;36m0.975871\u001b[0m  train   \n",
       "\u001b[1;36m4\u001b[0m          H3 Level \u001b[1;36m6\u001b[0m           eSPA  ERA5-Land + ArcticDEM  \u001b[1;36m0.540843\u001b[0m  train   \n",
       "..                \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m775\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m  Random Forest             AlphaEarth  \u001b[1;36m0.501394\u001b[0m   test   \n",
       "\u001b[1;36m776\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m        XGBoost             AlphaEarth  \u001b[1;36m0.824979\u001b[0m   test   \n",
       "\u001b[1;36m777\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m      AutoGluon             AlphaEarth  \u001b[1;36m0.515731\u001b[0m   test   \n",
       "\u001b[1;36m778\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m      AutoGluon             AlphaEarth  \u001b[1;36m0.510555\u001b[0m   test   \n",
       "\u001b[1;36m779\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m  Random Forest             AlphaEarth  \u001b[1;36m0.903151\u001b[0m   test   \n",
       "\n",
       "     metric  \n",
       "\u001b[1;36m0\u001b[0m        f1  \n",
       "\u001b[1;36m1\u001b[0m        f1  \n",
       "\u001b[1;36m2\u001b[0m        f1  \n",
       "\u001b[1;36m3\u001b[0m        f1  \n",
       "\u001b[1;36m4\u001b[0m        f1  \n",
       "..      \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m775\u001b[0m  recall  \n",
       "\u001b[1;36m776\u001b[0m  recall  \n",
       "\u001b[1;36m777\u001b[0m  recall  \n",
       "\u001b[1;36m778\u001b[0m  recall  \n",
       "\u001b[1;36m779\u001b[0m  recall  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m780\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bin_metrics = [\"f1\", \"precision\", \"recall\"]\n",
    "splits = [\"train\", \"test\"]\n",
    "# Create a long form DataFrame for easier plotting with seaborn\n",
    "binary_long = binary_runs.melt(\n",
    "    id_vars=[\"grid_level\", \"model_type\", \"experiment\"],\n",
    "    value_vars=[f\"{split}_{metric}\" for split in splits for metric in bin_metrics],\n",
    "    var_name=\"split_metric\",\n",
    "    value_name=\"score\",\n",
    ")\n",
    "# Split the \"split_metric\" column into separate \"split\" and \"metric\" columns\n",
    "binary_long[[\"split\", \"metric\"]] = binary_long[\"split_metric\"].str.rsplit(\"_\", n=1, expand=True)\n",
    "# Drop the original \"split_metric\" column\n",
    "binary_long = binary_long.drop(columns=[\"split_metric\"])\n",
    "binary_long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0e792a50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>experiment</th>\n",
       "      <th>score</th>\n",
       "      <th>split</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.880129</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.910010</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.782206</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.722951</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>kNN</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>kNN</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.600174</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.509789</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.642095</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.283430</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.504432</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>208 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "           grid_level     model_type             experiment     score  split  \\\n",
       "\u001b[1;36m0\u001b[0m          H3 Level \u001b[1;36m6\u001b[0m  Random Forest  ERA5-Land + ArcticDEM  \u001b[1;36m0.880129\u001b[0m  train   \n",
       "\u001b[1;36m1\u001b[0m          H3 Level \u001b[1;36m5\u001b[0m  Random Forest  ERA5-Land + ArcticDEM  \u001b[1;36m0.910010\u001b[0m  train   \n",
       "\u001b[1;36m2\u001b[0m          H3 Level \u001b[1;36m4\u001b[0m        XGBoost  ERA5-Land + ArcticDEM  \u001b[1;36m0.782206\u001b[0m  train   \n",
       "\u001b[1;36m3\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m  Random Forest  ERA5-Land + ArcticDEM  \u001b[1;36m0.722951\u001b[0m  train   \n",
       "\u001b[1;36m4\u001b[0m     HEALPix Level \u001b[1;36m7\u001b[0m            kNN  ERA5-Land + ArcticDEM  \u001b[1;36m1.000000\u001b[0m  train   \n",
       "..                \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m203\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m            kNN             AlphaEarth  \u001b[1;36m0.600174\u001b[0m   test   \n",
       "\u001b[1;36m204\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m        XGBoost             AlphaEarth  \u001b[1;36m0.509789\u001b[0m   test   \n",
       "\u001b[1;36m205\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m  Random Forest             AlphaEarth  \u001b[1;36m0.642095\u001b[0m   test   \n",
       "\u001b[1;36m206\u001b[0m   HEALPix Level \u001b[1;36m6\u001b[0m      AutoGluon             AlphaEarth  \u001b[1;36m0.283430\u001b[0m   test   \n",
       "\u001b[1;36m207\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m            kNN             AlphaEarth  \u001b[1;36m0.504432\u001b[0m   test   \n",
       "\n",
       "    metric  \n",
       "\u001b[1;36m0\u001b[0m       r2  \n",
       "\u001b[1;36m1\u001b[0m       r2  \n",
       "\u001b[1;36m2\u001b[0m       r2  \n",
       "\u001b[1;36m3\u001b[0m       r2  \n",
       "\u001b[1;36m4\u001b[0m       r2  \n",
       "..     \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m203\u001b[0m     r2  \n",
       "\u001b[1;36m204\u001b[0m     r2  \n",
       "\u001b[1;36m205\u001b[0m     r2  \n",
       "\u001b[1;36m206\u001b[0m     r2  \n",
       "\u001b[1;36m207\u001b[0m     r2  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m208\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "density_metrics = [\"r2\"]\n",
    "# Create a long form DataFrame for easier plotting with seaborn\n",
    "density_long = density_runs.melt(\n",
    "    id_vars=[\"grid_level\", \"model_type\", \"experiment\"],\n",
    "    value_vars=[f\"{split}_{metric}\" for split in splits for metric in density_metrics],\n",
    "    var_name=\"split_metric\",\n",
    "    value_name=\"score\",\n",
    ")\n",
    "# Split the \"split_metric\" column into separate \"split\" and \"metric\" columns\n",
    "density_long[[\"split\", \"metric\"]] = density_long[\"split_metric\"].str.split(\"_\", n=1, expand=True)\n",
    "# Drop the original \"split_metric\" column\n",
    "density_long = density_long.drop(columns=[\"split_metric\"])\n",
    "density_long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d200d56d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn.axisgrid.FacetGrid\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f46746c8f50\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 332068 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m1578.\u001b[0m\u001b[39m75x700 with \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_metrics_long = pd.concat([binary_long, density_long])\n",
    "g = sns.relplot(\n",
    "    pd.concat([binary_long, density_long]),\n",
    "    x=\"grid_level\",\n",
    "    y=\"score\",\n",
    "    col=\"metric\",\n",
    "    row=\"split\",\n",
    "    hue=\"model_type\",\n",
    "    kind=\"line\",\n",
    "    # style=\"experiment\",\n",
    "    hue_order=reversed([\"AutoGluon\", \"XGBoost\", \"Random Forest\", \"kNN\", \"eSPA\"]),\n",
    "    # style_order=reversed([\"AutoGluon\", \"XGBoost\", \"Random Forest\", \"kNN\", \"eSPA\"]),\n",
    "    markers=True,\n",
    "    height=3.5,\n",
    "    aspect=1,\n",
    ")\n",
    "for ax in g.axes.flat:\n",
    "    for label in ax.get_xticklabels():\n",
    "        # Align x-axis labels right and rotate them by 45 degrees\n",
    "        label.set_horizontalalignment(\"right\")\n",
    "        label.set_rotation(45)\n",
    "    # Titleize the axes title\n",
    "    title = ax.get_title()\n",
    "    title = (\n",
    "        title.replace(\"metric = \", \"\")\n",
    "        .replace(\"split = \", \"\")\n",
    "        .replace(\"train\", \"train-split\")\n",
    "        .replace(\"test\", \"test-split\")\n",
    "    )\n",
    "    ax.set_title(title.title())\n",
    "    # Limit all axes to (0, 1)\n",
    "    ax.set_ylim(0, 1)\n",
    "\n",
    "    # Draw a box signaling the grid-resolutions\n",
    "    ax.axvspan(-0.5, 3.5, color=\"#2e4a9e\", alpha=0.08)\n",
    "    ax.axvspan(3.5, 6.5, color=\"#005a00\", alpha=0.08)\n",
    "    ax.axvspan(6.5, 8.5, color=\"#6b4700\", alpha=0.08)\n",
    "\n",
    "    # Add text labels for the grid resolutions\n",
    "    ax.text(1.5, 0.05, \"Low\", color=\"#2e4a9e\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "    ax.text(5, 0.05, \"Medium\", color=\"#005a00\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "    ax.text(7.5, 0.05, \"High\", color=\"#6b4700\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "\n",
    "# Change the legend title to \"Model\"\n",
    "g._legend.set_title(\"Model\")\n",
    "\n",
    "# Change all the titles to be more descriptive\n",
    "g.set_axis_labels(\"Grid\", \"Score\")\n",
    "g.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f4c9e6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Model], [H3 Level 3], [H3 Level 4], [H3 Level 5], [H3 Level 6],\n",
      "[eSPA], [$0.915 plus.minus 0.028$], [$0.918 plus.minus 0.014$], [$0.777 plus.minus 0.071$], [$0.421 plus.minus 0.062$],\n",
      "[XGBoost], [$0.937 plus.minus 0.011$], [$0.929 plus.minus 0.002$], [$0.840 plus.minus 0.007$], [$0.577 plus.minus 0.010$],\n",
      "[Random Forest], [$0.940 plus.minus 0.007$], [$0.931 plus.minus 0.002$], [$0.842 plus.minus 0.008$], [$0.574 plus.minus 0.008$],\n",
      "[kNN], [$0.941 plus.minus 0.010$], [$0.929 plus.minus 0.004$], [$0.841 plus.minus 0.005$], [$0.568 plus.minus 0.017$],\n",
      "[AutoGluon], [$0.954 plus.minus 0.010$], [$0.929 plus.minus 0.003$], [$0.845 plus.minus 0.010$], [$0.574 plus.minus 0.017$],\n",
      "[All], [$0.937 plus.minus 0.020$], [$0.927 plus.minus 0.008$], [$0.829 plus.minus 0.042$], [$0.543 plus.minus 0.068$],\n",
      "[All (except eSPA)], [$0.943 plus.minus 0.012$], [$0.930 plus.minus 0.003$], [$0.842 plus.minus 0.008$], [$0.573 plus.minus 0.014$],\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make a typst table, which has the different grid-levels as columns and the different models as rows\n",
    "# Value used from the test split and the f1 metric for binary (Always the mean and std is calculated)\n",
    "typst_table = \"\"\n",
    "h3_levels = [f\"H3 Level {i}\" for i in range(3, 7)]\n",
    "# Header row\n",
    "typst_table += \"[Model], [\" + \"], [\".join(h3_levels) + \"],\\n\"\n",
    "for model in all_metrics_long[\"model_type\"].unique():\n",
    "    row = [model]\n",
    "    for grid_level in h3_levels:\n",
    "        value = all_metrics_long[\n",
    "            (all_metrics_long[\"model_type\"] == model)\n",
    "            & (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "            & (all_metrics_long[\"split\"] == \"test\")\n",
    "            & (all_metrics_long[\"metric\"] == \"f1\")\n",
    "        ][\"score\"].values\n",
    "        if len(value) > 0:\n",
    "            vmean = np.mean(value)\n",
    "            vstd = np.std(value)\n",
    "            row.append(f\"${vmean:.3f} plus.minus {vstd:.3f}$\")\n",
    "        else:\n",
    "            row.append(\"N/A\")\n",
    "    typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "# Add a final row with the mean across all models for each grid level\n",
    "row = [\"All\"]\n",
    "for grid_level in h3_levels:\n",
    "    value = all_metrics_long[\n",
    "        (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "        & (all_metrics_long[\"split\"] == \"test\")\n",
    "        & (all_metrics_long[\"metric\"] == \"f1\")\n",
    "    ][\"score\"].values\n",
    "    if len(value) > 0:\n",
    "        vmean = np.mean(value)\n",
    "        vstd = np.std(value)\n",
    "        row.append(f\"${vmean:.3f} plus.minus {vstd:.3f}$\")\n",
    "    else:\n",
    "        row.append(\"N/A\")\n",
    "typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "\n",
    "# Add another final row with the mean across all models for each grid, except the espa model\n",
    "row = [\"All (except eSPA)\"]\n",
    "for grid_level in h3_levels:\n",
    "    value = all_metrics_long[\n",
    "        (all_metrics_long[\"model_type\"] != \"eSPA\")\n",
    "        & (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "        & (all_metrics_long[\"split\"] == \"test\")\n",
    "        & (all_metrics_long[\"metric\"] == \"f1\")\n",
    "    ][\"score\"].values\n",
    "    if len(value) > 0:\n",
    "        vmean = np.mean(value)\n",
    "        vstd = np.std(value)\n",
    "        row.append(f\"${vmean:.3f} plus.minus {vstd:.3f}$\")\n",
    "    else:\n",
    "        row.append(\"N/A\")\n",
    "typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "print(typst_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13df6472",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Model], [H3 Level 3], [H3 Level 4], [H3 Level 5], [H3 Level 6],\n",
      "table_subheader(colspan: 5)[ERA5-Land + ArcticDEM],\n",
      "[@espa], [$0.947$], [$0.917$], [$0.817$], [$0.515$],\n",
      "[@xgboost], [$0.917$], [$0.930$], [$0.845$], [$0.589$],\n",
      "[@rf:long], [$0.927$], [$0.932$], [$0.846$], [$0.583$],\n",
      "[@knn], [$0.939$], [$0.924$], [$0.841$], [$0.570$],\n",
      "[AutoGluon], [$0.956$], [$0.930$], [$0.852$], [$0.590$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All (except eSPA)], [$0.935 plus.minus 0.014$], [$0.929 plus.minus 0.003$], [$0.846 plus.minus 0.004$], [$0.583 plus.minus 0.008$],\n",
      "table_subheader(colspan: 5)[ERA5-Land + ArcticDEM (Univariate Selection)],\n",
      "[@espa], [$0.923$], [$0.930$], [$0.810$], [$0.405$],\n",
      "[@xgboost], [$0.942$], [$0.931$], [$0.836$], [$0.561$],\n",
      "[@rf:long], [$0.942$], [$0.933$], [$0.839$], [$0.577$],\n",
      "[@knn], [$0.926$], [$0.932$], [$0.834$], [$0.553$],\n",
      "[AutoGluon], [$0.962$], [$0.933$], [$0.841$], [$0.546$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All (except eSPA)], [$0.943 plus.minus 0.013$], [$0.932 plus.minus 0.001$], [$0.837 plus.minus 0.002$], [$0.559 plus.minus 0.012$],\n",
      "table_subheader(colspan: 5)[ERA5-Land + ArcticDEM (Cluster Selection)],\n",
      "[@espa], [$0.917$], [$0.928$], [$0.825$], [$0.421$],\n",
      "[@xgboost], [$0.946$], [$0.930$], [$0.848$], [$0.579$],\n",
      "[@rf:long], [$0.946$], [$0.930$], [$0.852$], [$0.576$],\n",
      "[@knn], [$0.946$], [$0.926$], [$0.847$], [$0.556$],\n",
      "[AutoGluon], [$0.938$], [$0.928$], [$0.856$], [$0.585$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All (except eSPA)], [$0.944 plus.minus 0.004$], [$0.929 plus.minus 0.002$], [$0.851 plus.minus 0.004$], [$0.574 plus.minus 0.011$],\n",
      "table_subheader(colspan: 5)[AlphaEarth],\n",
      "[@espa], [$0.871$], [$0.895$], [$0.655$], [$0.342$],\n",
      "[@xgboost], [$0.942$], [$0.927$], [$0.830$], [$0.577$],\n",
      "[@rf:long], [$0.943$], [$0.929$], [$0.831$], [$0.561$],\n",
      "[@knn], [$0.951$], [$0.934$], [$0.841$], [$0.595$],\n",
      "[AutoGluon], [$0.962$], [$0.925$], [$0.831$], [$0.576$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All (except eSPA)], [$0.950 plus.minus 0.008$], [$0.928 plus.minus 0.003$], [$0.833 plus.minus 0.005$], [$0.577 plus.minus 0.012$],\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make a typst table, which has the different grid-levels as columns and the different models as rows\n",
    "# Value used from the test split and the f1 metric for binary (Always the mean and std is calculated)\n",
    "typst_table = \"\"\n",
    "h3_levels = [f\"H3 Level {i}\" for i in range(3, 7)]\n",
    "# Header row\n",
    "typst_table += \"[Model], [\" + \"], [\".join(h3_levels) + \"],\\n\"\n",
    "for experiment in all_metrics_long[\"experiment\"].unique():\n",
    "    typst_table += f\"table_subheader(colspan: 5)[{experiment}],\\n\"\n",
    "    for model in all_metrics_long[\"model_type\"].unique():\n",
    "        row = [\n",
    "            model.replace(\"eSPA\", \"@espa\")\n",
    "            .replace(\"XGBoost\", \"@xgboost\")\n",
    "            .replace(\"kNN\", \"@knn\")\n",
    "            .replace(\"Random Forest\", \"@rf:long\")\n",
    "        ]\n",
    "        for grid_level in h3_levels:\n",
    "            value = all_metrics_long[\n",
    "                (all_metrics_long[\"model_type\"] == model)\n",
    "                & (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "                & (all_metrics_long[\"split\"] == \"test\")\n",
    "                & (all_metrics_long[\"metric\"] == \"f1\")\n",
    "                & (all_metrics_long[\"experiment\"] == experiment)\n",
    "            ][\"score\"].values\n",
    "            assert len(value) == 1\n",
    "            row.append(f\"${value[0]:.3f}$\")\n",
    "        typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "    typst_table += \"table.hline(stroke: black + 0.5pt),\\n\"\n",
    "    # Add a final row with the mean across all models for each grid level\n",
    "    row = [\"All\"]\n",
    "    for grid_level in h3_levels:\n",
    "        value = all_metrics_long[\n",
    "            (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "            & (all_metrics_long[\"split\"] == \"test\")\n",
    "            & (all_metrics_long[\"metric\"] == \"f1\")\n",
    "            & (all_metrics_long[\"experiment\"] == experiment)\n",
    "        ][\"score\"].values\n",
    "        if len(value) > 0:\n",
    "            vmean = np.mean(value)\n",
    "            vstd = np.std(value)\n",
    "            row.append(f\"${vmean:.3f} plus.minus {vstd:.3f}$\")\n",
    "        else:\n",
    "            row.append(\"N/A\")\n",
    "    # typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "\n",
    "    # Add another final row with the mean across all models for each grid, except the espa model\n",
    "    row = [\"All (except eSPA)\"]\n",
    "    for grid_level in h3_levels:\n",
    "        value = all_metrics_long[\n",
    "            (all_metrics_long[\"model_type\"] != \"eSPA\")\n",
    "            & (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "            & (all_metrics_long[\"split\"] == \"test\")\n",
    "            & (all_metrics_long[\"metric\"] == \"f1\")\n",
    "            & (all_metrics_long[\"experiment\"] == experiment)\n",
    "        ][\"score\"].values\n",
    "        if len(value) > 0:\n",
    "            vmean = np.mean(value)\n",
    "            vstd = np.std(value)\n",
    "            row.append(f\"${vmean:.3f} plus.minus {vstd:.3f}$\")\n",
    "        else:\n",
    "            row.append(\"N/A\")\n",
    "    typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "print(typst_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "51b9dfb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Model], [H3 Level 3], [H3 Level 4], [H3 Level 5], [H3 Level 6],\n",
      "table_subheader(colspan: 5)[ERA5-Land + ArcticDEM],\n",
      "[@xgboost], [$0.023$], [$0.498$], [$0.557$], [$0.409$],\n",
      "[@rf:long], [$-0.020$], [$0.455$], [$0.577$], [$0.439$],\n",
      "[@knn], [$0.092$], [$0.323$], [$0.511$], [$0.395$],\n",
      "[AutoGluon], [$0.228$], [$0.556$], [$0.562$], [$0.404$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All], [$0.937 plus.minus 0.014$], [$0.927 plus.minus 0.006$], [$0.840 plus.minus 0.012$], [$0.569 plus.minus 0.028$],\n",
      "table_subheader(colspan: 5)[ERA5-Land + ArcticDEM (Univariate Selection)],\n",
      "[@xgboost], [$0.024$], [$0.438$], [$0.439$], [$0.396$],\n",
      "[@rf:long], [$0.068$], [$0.548$], [$0.515$], [$0.358$],\n",
      "[@knn], [$0.150$], [$0.493$], [$0.450$], [$0.375$],\n",
      "[AutoGluon], [$0.183$], [$0.617$], [$0.447$], [$0.396$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All], [$0.939 plus.minus 0.014$], [$0.932 plus.minus 0.001$], [$0.832 plus.minus 0.011$], [$0.528 plus.minus 0.063$],\n",
      "table_subheader(colspan: 5)[ERA5-Land + ArcticDEM (Cluster Selection)],\n",
      "[@xgboost], [$0.055$], [$0.393$], [$0.446$], [$0.411$],\n",
      "[@rf:long], [$0.118$], [$0.390$], [$0.409$], [$0.413$],\n",
      "[@knn], [$0.121$], [$0.275$], [$0.407$], [$0.386$],\n",
      "[AutoGluon], [$0.302$], [$0.448$], [$0.466$], [$0.411$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All], [$0.939 plus.minus 0.011$], [$0.929 plus.minus 0.002$], [$0.846 plus.minus 0.011$], [$0.544 plus.minus 0.062$],\n",
      "table_subheader(colspan: 5)[AlphaEarth],\n",
      "[@xgboost], [$-0.395$], [$0.414$], [$0.601$], [$0.490$],\n",
      "[@rf:long], [$-0.063$], [$0.425$], [$0.642$], [$0.499$],\n",
      "[@knn], [$0.014$], [$0.445$], [$0.483$], [$0.504$],\n",
      "[AutoGluon], [$0.104$], [$0.490$], [$0.682$], [$0.517$],\n",
      "table.hline(stroke: black + 0.5pt),\n",
      "[All], [$0.934 plus.minus 0.032$], [$0.922 plus.minus 0.014$], [$0.798 plus.minus 0.072$], [$0.530 plus.minus 0.094$],\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make a typst table, which has the different grid-levels as columns and the different models as rows\n",
    "# Value used from the test split and the f1 metric for binary (Always the mean and std is calculated)\n",
    "typst_table = \"\"\n",
    "h3_levels = [f\"H3 Level {i}\" for i in range(3, 7)]\n",
    "# Header row\n",
    "typst_table += \"[Model], [\" + \"], [\".join(h3_levels) + \"],\\n\"\n",
    "for experiment in all_metrics_long[\"experiment\"].unique():\n",
    "    typst_table += f\"table_subheader(colspan: 5)[{experiment}],\\n\"\n",
    "    for model in all_metrics_long[\"model_type\"].unique():\n",
    "        if model == \"eSPA\":\n",
    "            continue\n",
    "        row = [\n",
    "            model.replace(\"eSPA\", \"@espa\")\n",
    "            .replace(\"XGBoost\", \"@xgboost\")\n",
    "            .replace(\"kNN\", \"@knn\")\n",
    "            .replace(\"Random Forest\", \"@rf:long\")\n",
    "        ]\n",
    "        for grid_level in h3_levels:\n",
    "            value = all_metrics_long[\n",
    "                (all_metrics_long[\"model_type\"] == model)\n",
    "                & (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "                & (all_metrics_long[\"split\"] == \"test\")\n",
    "                & (all_metrics_long[\"metric\"] == \"r2\")\n",
    "                & (all_metrics_long[\"experiment\"] == experiment)\n",
    "            ][\"score\"].values\n",
    "            assert len(value) == 1\n",
    "            row.append(f\"${value[0]:.3f}$\")\n",
    "        typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "    typst_table += \"table.hline(stroke: black + 0.5pt),\\n\"\n",
    "    # Add a final row with the mean across all models for each grid level\n",
    "    row = [\"All\"]\n",
    "    for grid_level in h3_levels:\n",
    "        value = all_metrics_long[\n",
    "            (all_metrics_long[\"grid_level\"] == grid_level)\n",
    "            & (all_metrics_long[\"split\"] == \"test\")\n",
    "            & (all_metrics_long[\"metric\"] == \"f1\")\n",
    "            & (all_metrics_long[\"experiment\"] == experiment)\n",
    "        ][\"score\"].values\n",
    "        if len(value) > 0:\n",
    "            vmean = np.mean(value)\n",
    "            vstd = np.std(value)\n",
    "            row.append(f\"${vmean:.3f} plus.minus {vstd:.3f}$\")\n",
    "        else:\n",
    "            row.append(\"N/A\")\n",
    "    typst_table += \"[\" + \"], [\".join(row) + \"],\\n\"\n",
    "\n",
    "print(typst_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "37b6f336",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>experiment</th>\n",
       "      <th>score</th>\n",
       "      <th>split</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.905759</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.762885</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.959493</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.975871</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>0.540843</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>kNN</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.600174</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.509789</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.642095</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.283430</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>0.504432</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>988 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "           grid_level     model_type             experiment     score  split  \\\n",
       "\u001b[1;36m0\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m           eSPA  ERA5-Land + ArcticDEM  \u001b[1;36m0.905759\u001b[0m  train   \n",
       "\u001b[1;36m1\u001b[0m     HEALPix Level \u001b[1;36m9\u001b[0m           eSPA  ERA5-Land + ArcticDEM  \u001b[1;36m0.762885\u001b[0m  train   \n",
       "\u001b[1;36m2\u001b[0m          H3 Level \u001b[1;36m5\u001b[0m        XGBoost  ERA5-Land + ArcticDEM  \u001b[1;36m0.959493\u001b[0m  train   \n",
       "\u001b[1;36m3\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m  Random Forest  ERA5-Land + ArcticDEM  \u001b[1;36m0.975871\u001b[0m  train   \n",
       "\u001b[1;36m4\u001b[0m          H3 Level \u001b[1;36m6\u001b[0m           eSPA  ERA5-Land + ArcticDEM  \u001b[1;36m0.540843\u001b[0m  train   \n",
       "..                \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m203\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m            kNN             AlphaEarth  \u001b[1;36m0.600174\u001b[0m   test   \n",
       "\u001b[1;36m204\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m        XGBoost             AlphaEarth  \u001b[1;36m0.509789\u001b[0m   test   \n",
       "\u001b[1;36m205\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m  Random Forest             AlphaEarth  \u001b[1;36m0.642095\u001b[0m   test   \n",
       "\u001b[1;36m206\u001b[0m   HEALPix Level \u001b[1;36m6\u001b[0m      AutoGluon             AlphaEarth  \u001b[1;36m0.283430\u001b[0m   test   \n",
       "\u001b[1;36m207\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m            kNN             AlphaEarth  \u001b[1;36m0.504432\u001b[0m   test   \n",
       "\n",
       "    metric  \n",
       "\u001b[1;36m0\u001b[0m       f1  \n",
       "\u001b[1;36m1\u001b[0m       f1  \n",
       "\u001b[1;36m2\u001b[0m       f1  \n",
       "\u001b[1;36m3\u001b[0m       f1  \n",
       "\u001b[1;36m4\u001b[0m       f1  \n",
       "..     \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m203\u001b[0m     r2  \n",
       "\u001b[1;36m204\u001b[0m     r2  \n",
       "\u001b[1;36m205\u001b[0m     r2  \n",
       "\u001b[1;36m206\u001b[0m     r2  \n",
       "\u001b[1;36m207\u001b[0m     r2  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m988\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_metrics_long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b27cb607",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/3295687492.py:16: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  diff_summary = diff.groupby([\"grid_level\", \"metric\", \"split\"])[\"score_diff\"].agg([\"mean\", \"std\", \"min\", \"max\"]).reset_index()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>metric</th>\n",
       "      <th>split</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>0.014959</td>\n",
       "      <td>0.007703</td>\n",
       "      <td>0.006512</td>\n",
       "      <td>0.024876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.024672</td>\n",
       "      <td>0.020431</td>\n",
       "      <td>-0.047596</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.027466</td>\n",
       "      <td>0.007575</td>\n",
       "      <td>-0.036654</td>\n",
       "      <td>-0.019356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.000534</td>\n",
       "      <td>0.006984</td>\n",
       "      <td>-0.005388</td>\n",
       "      <td>0.009835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.010240</td>\n",
       "      <td>0.004506</td>\n",
       "      <td>-0.013707</td>\n",
       "      <td>-0.003773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.012767</td>\n",
       "      <td>0.009347</td>\n",
       "      <td>-0.021760</td>\n",
       "      <td>0.000371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>0.008375</td>\n",
       "      <td>0.012690</td>\n",
       "      <td>-0.002439</td>\n",
       "      <td>0.024732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.004618</td>\n",
       "      <td>0.023433</td>\n",
       "      <td>-0.025205</td>\n",
       "      <td>0.029091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.005667</td>\n",
       "      <td>0.021225</td>\n",
       "      <td>-0.022231</td>\n",
       "      <td>0.025456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "          grid_level metric split      mean       std       min       max\n",
       "\u001b[1;36m0\u001b[0m         H3 Level \u001b[1;36m3\u001b[0m     f1  test  \u001b[1;36m0.014959\u001b[0m  \u001b[1;36m0.007703\u001b[0m  \u001b[1;36m0.006512\u001b[0m  \u001b[1;36m0.024876\u001b[0m\n",
       "\u001b[1;36m8\u001b[0m    HEALPix Level \u001b[1;36m6\u001b[0m     f1  test \u001b[1;36m-0.024672\u001b[0m  \u001b[1;36m0.020431\u001b[0m \u001b[1;36m-0.047596\u001b[0m  \u001b[1;36m0.000000\u001b[0m\n",
       "\u001b[1;36m16\u001b[0m   HEALPix Level \u001b[1;36m7\u001b[0m     f1  test \u001b[1;36m-0.027466\u001b[0m  \u001b[1;36m0.007575\u001b[0m \u001b[1;36m-0.036654\u001b[0m \u001b[1;36m-0.019356\u001b[0m\n",
       "\u001b[1;36m24\u001b[0m        H3 Level \u001b[1;36m4\u001b[0m     f1  test \u001b[1;36m-0.000534\u001b[0m  \u001b[1;36m0.006984\u001b[0m \u001b[1;36m-0.005388\u001b[0m  \u001b[1;36m0.009835\u001b[0m\n",
       "\u001b[1;36m32\u001b[0m   HEALPix Level \u001b[1;36m8\u001b[0m     f1  test \u001b[1;36m-0.010240\u001b[0m  \u001b[1;36m0.004506\u001b[0m \u001b[1;36m-0.013707\u001b[0m \u001b[1;36m-0.003773\u001b[0m\n",
       "\u001b[1;36m40\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m     f1  test \u001b[1;36m-0.012767\u001b[0m  \u001b[1;36m0.009347\u001b[0m \u001b[1;36m-0.021760\u001b[0m  \u001b[1;36m0.000371\u001b[0m\n",
       "\u001b[1;36m48\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m     f1  test  \u001b[1;36m0.008375\u001b[0m  \u001b[1;36m0.012690\u001b[0m \u001b[1;36m-0.002439\u001b[0m  \u001b[1;36m0.024732\u001b[0m\n",
       "\u001b[1;36m56\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m     f1  test \u001b[1;36m-0.004618\u001b[0m  \u001b[1;36m0.023433\u001b[0m \u001b[1;36m-0.025205\u001b[0m  \u001b[1;36m0.029091\u001b[0m\n",
       "\u001b[1;36m64\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m     f1  test \u001b[1;36m-0.005667\u001b[0m  \u001b[1;36m0.021225\u001b[0m \u001b[1;36m-0.022231\u001b[0m  \u001b[1;36m0.025456\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/3295687492.py:16: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  diff_summary = diff.groupby([\"grid_level\", \"metric\", \"split\"])[\"score_diff\"].agg([\"mean\", \"std\", \"min\", \"max\"]).reset_index()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>metric</th>\n",
       "      <th>split</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>0.001563</td>\n",
       "      <td>0.024791</td>\n",
       "      <td>-0.029937</td>\n",
       "      <td>0.028997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>0.001983</td>\n",
       "      <td>0.005527</td>\n",
       "      <td>-0.002086</td>\n",
       "      <td>0.011397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>0.005295</td>\n",
       "      <td>0.001813</td>\n",
       "      <td>0.002995</td>\n",
       "      <td>0.007413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.025455</td>\n",
       "      <td>0.037950</td>\n",
       "      <td>-0.093106</td>\n",
       "      <td>-0.005056</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "          grid_level metric split      mean       std       min       max\n",
       "\u001b[1;36m0\u001b[0m         H3 Level \u001b[1;36m3\u001b[0m     f1  test  \u001b[1;36m0.001563\u001b[0m  \u001b[1;36m0.024791\u001b[0m \u001b[1;36m-0.029937\u001b[0m  \u001b[1;36m0.028997\u001b[0m\n",
       "\u001b[1;36m8\u001b[0m    HEALPix Level \u001b[1;36m6\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m16\u001b[0m   HEALPix Level \u001b[1;36m7\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m24\u001b[0m        H3 Level \u001b[1;36m4\u001b[0m     f1  test  \u001b[1;36m0.001983\u001b[0m  \u001b[1;36m0.005527\u001b[0m \u001b[1;36m-0.002086\u001b[0m  \u001b[1;36m0.011397\u001b[0m\n",
       "\u001b[1;36m32\u001b[0m   HEALPix Level \u001b[1;36m8\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m40\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m     f1  test  \u001b[1;36m0.005295\u001b[0m  \u001b[1;36m0.001813\u001b[0m  \u001b[1;36m0.002995\u001b[0m  \u001b[1;36m0.007413\u001b[0m\n",
       "\u001b[1;36m48\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m56\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m64\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m     f1  test \u001b[1;36m-0.025455\u001b[0m  \u001b[1;36m0.037950\u001b[0m \u001b[1;36m-0.093106\u001b[0m \u001b[1;36m-0.005056\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/3295687492.py:16: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  diff_summary = diff.groupby([\"grid_level\", \"metric\", \"split\"])[\"score_diff\"].agg([\"mean\", \"std\", \"min\", \"max\"]).reset_index()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>metric</th>\n",
       "      <th>split</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>0.001640</td>\n",
       "      <td>0.020183</td>\n",
       "      <td>-0.024291</td>\n",
       "      <td>0.024876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>0.005328</td>\n",
       "      <td>0.005079</td>\n",
       "      <td>0.001207</td>\n",
       "      <td>0.013006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.008433</td>\n",
       "      <td>0.002065</td>\n",
       "      <td>-0.011962</td>\n",
       "      <td>-0.006792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>f1</td>\n",
       "      <td>test</td>\n",
       "      <td>-0.040618</td>\n",
       "      <td>0.041063</td>\n",
       "      <td>-0.109559</td>\n",
       "      <td>-0.005430</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "          grid_level metric split      mean       std       min       max\n",
       "\u001b[1;36m0\u001b[0m         H3 Level \u001b[1;36m3\u001b[0m     f1  test  \u001b[1;36m0.001640\u001b[0m  \u001b[1;36m0.020183\u001b[0m \u001b[1;36m-0.024291\u001b[0m  \u001b[1;36m0.024876\u001b[0m\n",
       "\u001b[1;36m8\u001b[0m    HEALPix Level \u001b[1;36m6\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m16\u001b[0m   HEALPix Level \u001b[1;36m7\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m24\u001b[0m        H3 Level \u001b[1;36m4\u001b[0m     f1  test  \u001b[1;36m0.005328\u001b[0m  \u001b[1;36m0.005079\u001b[0m  \u001b[1;36m0.001207\u001b[0m  \u001b[1;36m0.013006\u001b[0m\n",
       "\u001b[1;36m32\u001b[0m   HEALPix Level \u001b[1;36m8\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m40\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m     f1  test \u001b[1;36m-0.008433\u001b[0m  \u001b[1;36m0.002065\u001b[0m \u001b[1;36m-0.011962\u001b[0m \u001b[1;36m-0.006792\u001b[0m\n",
       "\u001b[1;36m48\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m56\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m     f1  test       NaN       NaN       NaN       NaN\n",
       "\u001b[1;36m64\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m     f1  test \u001b[1;36m-0.040618\u001b[0m  \u001b[1;36m0.041063\u001b[0m \u001b[1;36m-0.109559\u001b[0m \u001b[1;36m-0.005430\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def _calc_diff_summary(experiment_name: str):\n",
    "    # Calculate the difference between the baseline and the Selection Method experiments\n",
    "    diff = all_metrics_long[all_metrics_long[\"experiment\"] == experiment_name].copy()\n",
    "    diff[\"experiment\"] = f\"{experiment_name} - ERA5-Land + ArcticDEM\"\n",
    "    baseline = all_metrics_long[all_metrics_long[\"experiment\"] == \"ERA5-Land + ArcticDEM\"]\n",
    "    diff = diff.merge(\n",
    "        baseline,\n",
    "        on=[\"grid_level\", \"model_type\", \"split\", \"metric\"],\n",
    "        suffixes=(\"_alphaearth\", \"_baseline\"),\n",
    "    )\n",
    "    diff[\"score_diff\"] = diff[\"score_alphaearth\"] - diff[\"score_baseline\"]\n",
    "    # If AlphaEarth, filter out eSPA model\n",
    "    if experiment_name == \"AlphaEarth\":\n",
    "        diff = diff[diff[\"model_type\"] != \"eSPA\"]\n",
    "    # Calculate the mean and std of the difference for each grid level\n",
    "    diff_summary = (\n",
    "        diff.groupby([\"grid_level\", \"metric\", \"split\"])[\"score_diff\"].agg([\"mean\", \"std\", \"min\", \"max\"]).reset_index()\n",
    "    )\n",
    "    return diff_summary[(diff_summary[\"metric\"] == \"f1\") & (diff_summary[\"split\"] == \"test\")]\n",
    "\n",
    "\n",
    "display(_calc_diff_summary(\"AlphaEarth\"))\n",
    "display(_calc_diff_summary(\"ERA5-Land + ArcticDEM (Cluster Selection)\"))\n",
    "display(_calc_diff_summary(\"ERA5-Land + ArcticDEM (Univariate Selection)\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0048852",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#table(\n",
      "  columns: 4,\n",
      "  table.header[grid_level][min][max][mean_plus_minus_std],\n",
      "  [H3 Level 3], [$0.007$], [$0.025$], [$0.015 plus.minus 0.008$],\n",
      "  [HEALPix Level 6], [$-0.048$], [$0.000$], [$-0.025 plus.minus 0.020$],\n",
      "  [HEALPix Level 7], [$-0.037$], [$-0.019$], [$-0.027 plus.minus 0.008$],\n",
      "  [H3 Level 4], [$-0.005$], [$0.010$], [$-0.001 plus.minus 0.007$],\n",
      "  [HEALPix Level 8], [$-0.014$], [$-0.004$], [$-0.010 plus.minus 0.005$],\n",
      "  [H3 Level 5], [$-0.022$], [$0.000$], [$-0.013 plus.minus 0.009$],\n",
      "  [HEALPix Level 9], [$-0.002$], [$0.025$], [$0.008 plus.minus 0.013$],\n",
      "  [HEALPix Level 10], [$-0.025$], [$0.029$], [$-0.005 plus.minus 0.023$],\n",
      "  [H3 Level 6], [$-0.022$], [$0.025$], [$-0.006 plus.minus 0.021$]\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/3295687492.py:16: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  diff_summary = diff.groupby([\"grid_level\", \"metric\", \"split\"])[\"score_diff\"].agg([\"mean\", \"std\", \"min\", \"max\"]).reset_index()\n"
     ]
    }
   ],
   "source": [
    "import pypst\n",
    "\n",
    "\n",
    "def _to_typst(diff_summary: pd.DataFrame) -> str:\n",
    "    diff_summary = diff_summary.drop(columns=[\"metric\", \"split\"]).copy()\n",
    "    # Drop all rows containing nans\n",
    "    diff_summary = diff_summary.dropna()\n",
    "    # Add a new column with the mean and std in the format \"mean plus.minus std\"\n",
    "    diff_summary[\"mean_plus_minus_std\"] = diff_summary.apply(\n",
    "        lambda row: f\"${row['mean']:.3f} plus.minus {row['std']:.3f}$\", axis=1\n",
    "    )\n",
    "    diff_summary = diff_summary.drop(columns=[\"mean\", \"std\"])\n",
    "    # Also format the min and max values\n",
    "    diff_summary[\"min\"] = diff_summary[\"min\"].apply(lambda x: f\"${x:.3f}$\")\n",
    "    diff_summary[\"max\"] = diff_summary[\"max\"].apply(lambda x: f\"${x:.3f}$\")\n",
    "    table = pypst.Table.from_dataframe(diff_summary, include_index=False)\n",
    "    return table.render()\n",
    "\n",
    "\n",
    "print(_to_typst(_calc_diff_summary(\"AlphaEarth\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "38396b2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#table(\n",
      "  columns: 4,\n",
      "  table.header[grid_level][min][max][mean_plus_minus_std],\n",
      "  [H3 Level 3], [$-0.030$], [$0.029$], [$0.002 plus.minus 0.025$],\n",
      "  [H3 Level 4], [$-0.002$], [$0.011$], [$0.002 plus.minus 0.006$],\n",
      "  [H3 Level 5], [$0.003$], [$0.007$], [$0.005 plus.minus 0.002$],\n",
      "  [H3 Level 6], [$-0.093$], [$-0.005$], [$-0.025 plus.minus 0.038$]\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/3295687492.py:16: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  diff_summary = diff.groupby([\"grid_level\", \"metric\", \"split\"])[\"score_diff\"].agg([\"mean\", \"std\", \"min\", \"max\"]).reset_index()\n"
     ]
    }
   ],
   "source": [
    "print(_to_typst(_calc_diff_summary(\"ERA5-Land + ArcticDEM (Cluster Selection)\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "b102ab10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#table(\n",
      "  columns: 4,\n",
      "  table.header[grid_level][min][max][mean_plus_minus_std],\n",
      "  [H3 Level 3], [$-0.030$], [$0.029$], [$0.002 plus.minus 0.025$],\n",
      "  [H3 Level 4], [$-0.002$], [$0.011$], [$0.002 plus.minus 0.006$],\n",
      "  [H3 Level 5], [$0.003$], [$0.007$], [$0.005 plus.minus 0.002$],\n",
      "  [H3 Level 6], [$-0.093$], [$-0.005$], [$-0.025 plus.minus 0.038$]\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/3295687492.py:16: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  diff_summary = diff.groupby([\"grid_level\", \"metric\", \"split\"])[\"score_diff\"].agg([\"mean\", \"std\", \"min\", \"max\"]).reset_index()\n"
     ]
    }
   ],
   "source": [
    "print(_to_typst(_calc_diff_summary(\"ERA5-Land + ArcticDEM (Cluster Selection)\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "5291a567",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 165944 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 175\u001b[0m\u001b[1;36m0x350\u001b[0m\u001b[39m with \u001b[0m\u001b[1;36m5\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Closeup F1 score on Test\n",
    "g = sns.relplot(\n",
    "    binary_long[(binary_long[\"metric\"] == \"f1\") & (binary_long[\"split\"] == \"test\")],\n",
    "    x=\"grid_level\",\n",
    "    y=\"score\",\n",
    "    hue=\"experiment\",\n",
    "    style=\"experiment\",\n",
    "    col=\"model_type\",\n",
    "    height=3.5,\n",
    "    aspect=1,\n",
    "    kind=\"line\",\n",
    "    col_wrap=5,\n",
    "    facet_kws={\"legend_out\": False},\n",
    ")\n",
    "for ax in g.axes.flat:\n",
    "    for label in ax.get_xticklabels():\n",
    "        # Align x-axis labels right and rotate them by 45 degrees\n",
    "        label.set_horizontalalignment(\"right\")\n",
    "        label.set_rotation(45)\n",
    "    # Titleize the axes title\n",
    "    title = ax.get_title()\n",
    "    title = title.replace(\"model_type = \", \"\")\n",
    "    ax.set_title(title.title())\n",
    "\n",
    "    # Limit all axes to (0.3, 1)\n",
    "    ax.set_ylim(0.3, 1)\n",
    "\n",
    "    # Draw a box signaling the grid-resolutions\n",
    "    ax.axvspan(-0.5, 3.5, color=\"#2e4a9e\", alpha=0.08)\n",
    "    ax.axvspan(3.5, 6.5, color=\"#005a00\", alpha=0.08)\n",
    "    ax.axvspan(6.5, 8.5, color=\"#6b4700\", alpha=0.08)\n",
    "\n",
    "    # Add text labels for the grid resolutions\n",
    "    ax.text(1.5, 0.35, \"Low\", color=\"#2e4a9e\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "    ax.text(5, 0.35, \"Medium\", color=\"#005a00\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "    ax.text(7.5, 0.35, \"High\", color=\"#6b4700\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "\n",
    "# Change the legend title to \"Model\"\n",
    "g._legend.set_title(\"Experiment\")\n",
    "\n",
    "# Change all the titles to be more descriptive\n",
    "g.set_axis_labels(\"Grid\", \"Score\")\n",
    "\n",
    "# Move the legend to the bottom\n",
    "sns.move_legend(g, \"upper center\", ncol=2, title=\"Experiment\", frameon=False, bbox_to_anchor=(0.5, 1.3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceabcdf5",
   "metadata": {},
   "source": [
    "## Feature Importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "8619a48e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def _parse_feature_name(feature_name: str) -> str:\n",
    "    feature_name = (\n",
    "        feature_name.replace(\"_median\", \"_\")\n",
    "        .replace(\"_mean\", \"_\")\n",
    "        .replace(\"era5_\", \"\")\n",
    "        .replace(\"arcticdem_\", \"\")\n",
    "        .replace(\"JFM\", \"Winter\")\n",
    "        .replace(\"AMJ\", \"Spring\")\n",
    "        .replace(\"JAS\", \"Summer\")\n",
    "        .replace(\"OND\", \"Fall\")\n",
    "        .replace(\"embeddings_embeddings\", \"embeddings\")\n",
    "        .replace(\"_\", \" \")\n",
    "        .title()\n",
    "    )\n",
    "    # Remove duplicate spaces\n",
    "    feature_name = \" \".join(feature_name.split())\n",
    "    return feature_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "b062bccf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>importance_std</th>\n",
       "      <th>importance_stability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Land Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Water Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>X</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29092</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Trend A59</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29093</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Trend A60</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29094</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Trend A61</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29095</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Trend A62</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29096</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Trend A63</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29097 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                  experiment  grid_level model_type     task  \\\n",
       "\u001b[1;36m0\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m1\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m2\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m3\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m4\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[33m...\u001b[0m                      \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m29092\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m29093\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m29094\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m29095\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m29096\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\n",
       "                    feature  importance  importance_std  importance_stability  \n",
       "\u001b[1;36m0\u001b[0m                 Cell Area    \u001b[1;36m0.000000\u001b[0m             \u001b[1;36m0.0\u001b[0m                   NaN  \n",
       "\u001b[1;36m1\u001b[0m                 Land Area    \u001b[1;36m0.000000\u001b[0m             \u001b[1;36m0.0\u001b[0m                   NaN  \n",
       "\u001b[1;36m2\u001b[0m                Water Area    \u001b[1;36m0.000000\u001b[0m             \u001b[1;36m0.0\u001b[0m                   NaN  \n",
       "\u001b[1;36m3\u001b[0m                Land Ratio    \u001b[1;36m0.000000\u001b[0m             \u001b[1;36m0.0\u001b[0m                   NaN  \n",
       "\u001b[1;36m4\u001b[0m                         X    \u001b[1;36m0.000000\u001b[0m             \u001b[1;36m0.0\u001b[0m                   NaN  \n",
       "\u001b[33m...\u001b[0m                     \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m                   \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m29092\u001b[0m  Embeddings Trend A59   \u001b[1;36m-0.059563\u001b[0m             \u001b[1;36m0.0\u001b[0m                   \u001b[1;36m0.0\u001b[0m  \n",
       "\u001b[1;36m29093\u001b[0m  Embeddings Trend A60   \u001b[1;36m-0.059563\u001b[0m             \u001b[1;36m0.0\u001b[0m                   \u001b[1;36m0.0\u001b[0m  \n",
       "\u001b[1;36m29094\u001b[0m  Embeddings Trend A61   \u001b[1;36m-0.059563\u001b[0m             \u001b[1;36m0.0\u001b[0m                   \u001b[1;36m0.0\u001b[0m  \n",
       "\u001b[1;36m29095\u001b[0m  Embeddings Trend A62   \u001b[1;36m-0.059563\u001b[0m             \u001b[1;36m0.0\u001b[0m                   \u001b[1;36m0.0\u001b[0m  \n",
       "\u001b[1;36m29096\u001b[0m  Embeddings Trend A63   \u001b[1;36m-0.059563\u001b[0m             \u001b[1;36m0.0\u001b[0m                   \u001b[1;36m0.0\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m29097\u001b[0m rows x \u001b[1;36m8\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create dataframe in long format for the feature importances\n",
    "# Cols: grid_level, model_type, task, feature, importance, stddev\n",
    "feature_importances = []\n",
    "for experiment_name in all_experiments[\"experiment\"].unique():\n",
    "    all_runs = all_experiments[all_experiments[\"experiment\"] == experiment_name]\n",
    "    for run in all_runs[\"run\"]:\n",
    "        task = run.task\n",
    "        grid = run.dataset.grid.replace(\"hex\", \"H3\").replace(\"healpix\", \"HEALPix\")\n",
    "        grid_level = f\"{grid} Level {run.dataset.level}\"\n",
    "        model_type = (\n",
    "            run.model_type.replace(\"rf\", \"Random Forest\")\n",
    "            .replace(\"xgboost\", \"XGBoost\")\n",
    "            .replace(\"knn\", \"kNN\")\n",
    "            .replace(\"espa\", \"eSPA\")\n",
    "            .replace(\"autogluon\", \"AutoGluon\")\n",
    "        )\n",
    "        importances = run.feature_importance[\"importance\"]\n",
    "        importances_std = run.feature_importance[\"stddev\"]\n",
    "        for feature, importance in importances.items():\n",
    "            feature_importances.append(\n",
    "                {\n",
    "                    \"experiment\": experiment_name,\n",
    "                    \"grid_level\": grid_level,\n",
    "                    \"model_type\": model_type,\n",
    "                    \"task\": task,\n",
    "                    \"feature\": feature,\n",
    "                    \"importance\": importance,\n",
    "                    \"importance_std\": importances_std.get(feature, 0.0),\n",
    "                }\n",
    "            )\n",
    "feature_importances = pd.DataFrame(feature_importances)\n",
    "feature_importances[\"grid_level\"] = feature_importances[\"grid_level\"].astype(grid_level_cat)\n",
    "\n",
    "\n",
    "feature_importances[\"feature\"] = feature_importances[\"feature\"].apply(_parse_feature_name)\n",
    "\n",
    "feature_importances_binary = feature_importances[feature_importances[\"task\"] == \"binary\"]\n",
    "feature_importances_density = feature_importances[feature_importances[\"task\"] == \"density\"]\n",
    "\n",
    "feature_importances[\"importance_stability\"] = (\n",
    "    feature_importances[\"importance_std\"] / (feature_importances[\"importance\"]).abs()\n",
    ")\n",
    "\n",
    "feature_importances"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe79ffc5",
   "metadata": {},
   "source": [
    "## SHAP values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "566c81b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 52 SHAP explanations for XGBoost models.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>importance_std</th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>explanation_number</th>\n",
       "      <th>importance_stability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cell Area</td>\n",
       "      <td>1.078092e-01</td>\n",
       "      <td>0.124502</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>1.154834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Land Area</td>\n",
       "      <td>1.206793e-02</td>\n",
       "      <td>0.036949</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>3.061753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Water Area</td>\n",
       "      <td>6.767041e-03</td>\n",
       "      <td>0.010750</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>1.588574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>1.002347e-02</td>\n",
       "      <td>0.019852</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>1.980514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>X</td>\n",
       "      <td>4.361002e-02</td>\n",
       "      <td>0.037302</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>0.855361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6461</th>\n",
       "      <td>Embeddings Trend A59</td>\n",
       "      <td>8.571458e-07</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "      <td>227</td>\n",
       "      <td>6.463868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6462</th>\n",
       "      <td>Embeddings Trend A60</td>\n",
       "      <td>1.483823e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "      <td>227</td>\n",
       "      <td>2.742164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6463</th>\n",
       "      <td>Embeddings Trend A61</td>\n",
       "      <td>1.398119e-06</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "      <td>227</td>\n",
       "      <td>10.943297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6464</th>\n",
       "      <td>Embeddings Trend A62</td>\n",
       "      <td>5.685622e-07</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "      <td>227</td>\n",
       "      <td>2.753818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6465</th>\n",
       "      <td>Embeddings Trend A63</td>\n",
       "      <td>1.334056e-06</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "      <td>227</td>\n",
       "      <td>2.532331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6466 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                   feature    importance  importance_std  \\\n",
       "\u001b[1;36m0\u001b[0m                Cell Area  \u001b[1;36m1.078092e-01\u001b[0m        \u001b[1;36m0.124502\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                Land Area  \u001b[1;36m1.206793e-02\u001b[0m        \u001b[1;36m0.036949\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m               Water Area  \u001b[1;36m6.767041e-03\u001b[0m        \u001b[1;36m0.010750\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m               Land Ratio  \u001b[1;36m1.002347e-02\u001b[0m        \u001b[1;36m0.019852\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                        X  \u001b[1;36m4.361002e-02\u001b[0m        \u001b[1;36m0.037302\u001b[0m   \n",
       "\u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m           \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m6461\u001b[0m  Embeddings Trend A59  \u001b[1;36m8.571458e-07\u001b[0m        \u001b[1;36m0.000006\u001b[0m   \n",
       "\u001b[1;36m6462\u001b[0m  Embeddings Trend A60  \u001b[1;36m1.483823e-06\u001b[0m        \u001b[1;36m0.000004\u001b[0m   \n",
       "\u001b[1;36m6463\u001b[0m  Embeddings Trend A61  \u001b[1;36m1.398119e-06\u001b[0m        \u001b[1;36m0.000015\u001b[0m   \n",
       "\u001b[1;36m6464\u001b[0m  Embeddings Trend A62  \u001b[1;36m5.685622e-07\u001b[0m        \u001b[1;36m0.000002\u001b[0m   \n",
       "\u001b[1;36m6465\u001b[0m  Embeddings Trend A63  \u001b[1;36m1.334056e-06\u001b[0m        \u001b[1;36m0.000003\u001b[0m   \n",
       "\n",
       "                 experiment        grid_level model_type     task  \\\n",
       "\u001b[1;36m0\u001b[0m     ERA5-Land + ArcticDEM        H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary   \n",
       "\u001b[1;36m1\u001b[0m     ERA5-Land + ArcticDEM        H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary   \n",
       "\u001b[1;36m2\u001b[0m     ERA5-Land + ArcticDEM        H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary   \n",
       "\u001b[1;36m3\u001b[0m     ERA5-Land + ArcticDEM        H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary   \n",
       "\u001b[1;36m4\u001b[0m     ERA5-Land + ArcticDEM        H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary   \n",
       "\u001b[33m...\u001b[0m                     \u001b[33m...\u001b[0m               \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m6461\u001b[0m             AlphaEarth  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density   \n",
       "\u001b[1;36m6462\u001b[0m             AlphaEarth  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density   \n",
       "\u001b[1;36m6463\u001b[0m             AlphaEarth  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density   \n",
       "\u001b[1;36m6464\u001b[0m             AlphaEarth  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density   \n",
       "\u001b[1;36m6465\u001b[0m             AlphaEarth  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density   \n",
       "\n",
       "      explanation_number  importance_stability  \n",
       "\u001b[1;36m0\u001b[0m                      \u001b[1;36m2\u001b[0m              \u001b[1;36m1.154834\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m                      \u001b[1;36m2\u001b[0m              \u001b[1;36m3.061753\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m                      \u001b[1;36m2\u001b[0m              \u001b[1;36m1.588574\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m                      \u001b[1;36m2\u001b[0m              \u001b[1;36m1.980514\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m                      \u001b[1;36m2\u001b[0m              \u001b[1;36m0.855361\u001b[0m  \n",
       "\u001b[33m...\u001b[0m                  \u001b[33m...\u001b[0m                   \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m6461\u001b[0m                 \u001b[1;36m227\u001b[0m              \u001b[1;36m6.463868\u001b[0m  \n",
       "\u001b[1;36m6462\u001b[0m                 \u001b[1;36m227\u001b[0m              \u001b[1;36m2.742164\u001b[0m  \n",
       "\u001b[1;36m6463\u001b[0m                 \u001b[1;36m227\u001b[0m             \u001b[1;36m10.943297\u001b[0m  \n",
       "\u001b[1;36m6464\u001b[0m                 \u001b[1;36m227\u001b[0m              \u001b[1;36m2.753818\u001b[0m  \n",
       "\u001b[1;36m6465\u001b[0m                 \u001b[1;36m227\u001b[0m              \u001b[1;36m2.532331\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m6466\u001b[0m rows x \u001b[1;36m9\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shap_importances = []\n",
    "shap_explanation = {}\n",
    "\n",
    "for i, row in all_experiments.iterrows():\n",
    "    if row.model_type != \"XGBoost\":\n",
    "        continue\n",
    "    run: Training = row.run\n",
    "    run.shap_explanation.feature_names = [_parse_feature_name(f) for f in run.shap_explanation.feature_names]\n",
    "    shap_explanation[i] = run.shap_explanation\n",
    "    shap_importances.append(\n",
    "        pd.DataFrame(\n",
    "            {\n",
    "                \"feature\": run.shap_explanation.feature_names,\n",
    "                \"importance\": np.abs(run.shap_explanation.values).mean(0),\n",
    "                \"importance_std\": np.abs(run.shap_explanation.values).std(0),\n",
    "                \"experiment\": row.experiment,\n",
    "                \"grid_level\": row.grid_level,\n",
    "                \"model_type\": row.model_type,\n",
    "                \"task\": row.task,\n",
    "                \"explanation_number\": i,\n",
    "            }\n",
    "        )\n",
    "    )\n",
    "\n",
    "print(f\"Found {len(shap_explanation)} SHAP explanations for XGBoost models.\")\n",
    "shap_importances = pd.concat(shap_importances, ignore_index=True)\n",
    "shap_importances[\"grid_level\"] = shap_importances[\"grid_level\"].astype(grid_level_cat)\n",
    "shap_importances[\"importance_stability\"] = shap_importances[\"importance_std\"] / (shap_importances[\"importance\"]).abs()\n",
    "shap_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "d8e66f75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 4 ERA5-Land + ArcticDEM.\n",
      "Found 4 ERA5-Land + ArcticDEM (Univariate Selection).\n",
      "Found 4 ERA5-Land + ArcticDEM (Cluster Selection).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "\u001b[1m(\u001b[0m\n",
       "    \u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 320\u001b[0m\u001b[1;36m0x400\u001b[0m\u001b[39m with \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m Axes>,\u001b[0m\n",
       "\u001b[39m    \u001b[0m\u001b[1;35marray\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m[\u001b[0m\u001b[39m<Axes: \u001b[0m\u001b[33mtitle\u001b[0m\u001b[39m=\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'center'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'SHAP Beeswarm for H3 Level 3'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m, \u001b[0m\u001b[33mxlabel\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'SHAP value \u001b[0m\u001b[32m(\u001b[0m\u001b[32mimpact on model output\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m\u001b[39m>,\u001b[0m\n",
       "\u001b[39m       <Axes: \u001b[0m\u001b[33mtitle\u001b[0m\u001b[39m=\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'center'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'SHAP Beeswarm for H3 Level 4'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m, \u001b[0m\u001b[33mxlabel\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'SHAP value \u001b[0m\u001b[32m(\u001b[0m\u001b[32mimpact on model output\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m\u001b[39m>,\u001b[0m\n",
       "\u001b[39m       <Axes: \u001b[0m\u001b[33mtitle\u001b[0m\u001b[39m=\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'center'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'SHAP Beeswarm for H3 Level 5'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m, \u001b[0m\u001b[33mxlabel\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'SHAP value \u001b[0m\u001b[32m(\u001b[0m\u001b[32mimpact on model output\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m\u001b[39m>,\u001b[0m\n",
       "\u001b[39m       <Axes: \u001b[0m\u001b[33mtitle\u001b[0m\u001b[39m=\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'center'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'SHAP Beeswarm for H3 Level 6'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m, \u001b[0m\u001b[33mxlabel\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'SHAP value \u001b[0m\u001b[32m(\u001b[0m\u001b[32mimpact on model output\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m\u001b[1m>\u001b[0m\u001b[1m]\u001b[0m,\n",
       "      \u001b[33mdtype\u001b[0m=\u001b[35mobject\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m)\u001b[0m"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 311584 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 320\u001b[0m\u001b[1;36m0x400\u001b[0m\u001b[39m with \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 323072 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 320\u001b[0m\u001b[1;36m0x400\u001b[0m\u001b[39m with \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 337344 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 320\u001b[0m\u001b[1;36m0x400\u001b[0m\u001b[39m with \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "is_binary = shap_importances[\"task\"] == \"binary\"\n",
    "is_hex_grid = shap_importances[\"grid_level\"].str.startswith(\"H3\")\n",
    "is_baseline = shap_importances[\"experiment\"] == \"ERA5-Land + ArcticDEM\"\n",
    "is_univariate = shap_importances[\"experiment\"] == \"ERA5-Land + ArcticDEM (Univariate Selection)\"\n",
    "is_cluster = shap_importances[\"experiment\"] == \"ERA5-Land + ArcticDEM (Cluster Selection)\"\n",
    "baseline_binary_explanation_numbers = shap_importances[is_binary & is_hex_grid & is_baseline][\n",
    "    \"explanation_number\"\n",
    "].unique()\n",
    "print(f\"Found {len(baseline_binary_explanation_numbers)} ERA5-Land + ArcticDEM.\")\n",
    "univariate_binary_explanation_numbers = shap_importances[is_binary & is_hex_grid & is_univariate][\n",
    "    \"explanation_number\"\n",
    "].unique()\n",
    "print(f\"Found {len(univariate_binary_explanation_numbers)} ERA5-Land + ArcticDEM (Univariate Selection).\")\n",
    "cluster_binary_explanation_numbers = shap_importances[is_binary & is_hex_grid & is_cluster][\n",
    "    \"explanation_number\"\n",
    "].unique()\n",
    "print(f\"Found {len(cluster_binary_explanation_numbers)} ERA5-Land + ArcticDEM (Cluster Selection).\")\n",
    "\n",
    "grid_level_order = (\n",
    "    shap_importances[shap_importances[\"grid_level\"].str.startswith(\"H3\")][\"grid_level\"]\n",
    "    .cat.remove_unused_categories()\n",
    "    .cat.categories.tolist()\n",
    ")\n",
    "\n",
    "\n",
    "def plot_shap_beeswarms(explanation_numbers: list[int]):  # noqa: D103\n",
    "    fig, axs = plt.subplots(nrows=1, ncols=len(explanation_numbers), figsize=(len(explanation_numbers) * 8, 4))\n",
    "    axs = axs.flatten()\n",
    "    for explanation_number in explanation_numbers:\n",
    "        grid_level = shap_importances[shap_importances[\"explanation_number\"] == explanation_number][\"grid_level\"].iloc[\n",
    "            0\n",
    "        ]\n",
    "        i = grid_level_order.index(grid_level)\n",
    "        shap.plots.beeswarm(shap_explanation[explanation_number], ax=axs[i], plot_size=None, show=False)\n",
    "        axs[i].set_title(f\"SHAP Beeswarm for {grid_level}\")\n",
    "    fig.tight_layout()\n",
    "    return fig, axs\n",
    "\n",
    "\n",
    "plot_shap_beeswarms(baseline_binary_explanation_numbers)\n",
    "plot_shap_beeswarms(univariate_binary_explanation_numbers)\n",
    "plot_shap_beeswarms(cluster_binary_explanation_numbers)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "f4a41b66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>importance_std</th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>explanation_number</th>\n",
       "      <th>importance_stability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cell Area</td>\n",
       "      <td>0.107809</td>\n",
       "      <td>0.124502</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>1.154834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Land Area</td>\n",
       "      <td>0.012068</td>\n",
       "      <td>0.036949</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>3.061753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Water Area</td>\n",
       "      <td>0.006767</td>\n",
       "      <td>0.010750</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>1.588574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>0.010023</td>\n",
       "      <td>0.019852</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>1.980514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>X</td>\n",
       "      <td>0.043610</td>\n",
       "      <td>0.037302</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>2</td>\n",
       "      <td>0.855361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4049</th>\n",
       "      <td>Sshf Trend Spring</td>\n",
       "      <td>0.018916</td>\n",
       "      <td>0.013831</td>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>147</td>\n",
       "      <td>0.731204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4050</th>\n",
       "      <td>Sshf Spring</td>\n",
       "      <td>0.012886</td>\n",
       "      <td>0.017457</td>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>147</td>\n",
       "      <td>1.354661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4051</th>\n",
       "      <td>Precipitation Occurrences Trend Fall</td>\n",
       "      <td>0.064147</td>\n",
       "      <td>0.059612</td>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>147</td>\n",
       "      <td>0.929308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4052</th>\n",
       "      <td>Effective Snow Depth Trend Fall</td>\n",
       "      <td>0.057669</td>\n",
       "      <td>0.064099</td>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>147</td>\n",
       "      <td>1.111485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4053</th>\n",
       "      <td>Snowc Trend Fall</td>\n",
       "      <td>0.073745</td>\n",
       "      <td>0.043125</td>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "      <td>147</td>\n",
       "      <td>0.584792</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2027 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                   feature  importance  importance_std  \\\n",
       "\u001b[1;36m0\u001b[0m                                Cell Area    \u001b[1;36m0.107809\u001b[0m        \u001b[1;36m0.124502\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                                Land Area    \u001b[1;36m0.012068\u001b[0m        \u001b[1;36m0.036949\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m                               Water Area    \u001b[1;36m0.006767\u001b[0m        \u001b[1;36m0.010750\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                               Land Ratio    \u001b[1;36m0.010023\u001b[0m        \u001b[1;36m0.019852\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                                        X    \u001b[1;36m0.043610\u001b[0m        \u001b[1;36m0.037302\u001b[0m   \n",
       "\u001b[33m...\u001b[0m                                    \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m4049\u001b[0m                     Sshf Trend Spring    \u001b[1;36m0.018916\u001b[0m        \u001b[1;36m0.013831\u001b[0m   \n",
       "\u001b[1;36m4050\u001b[0m                           Sshf Spring    \u001b[1;36m0.012886\u001b[0m        \u001b[1;36m0.017457\u001b[0m   \n",
       "\u001b[1;36m4051\u001b[0m  Precipitation Occurrences Trend Fall    \u001b[1;36m0.064147\u001b[0m        \u001b[1;36m0.059612\u001b[0m   \n",
       "\u001b[1;36m4052\u001b[0m       Effective Snow Depth Trend Fall    \u001b[1;36m0.057669\u001b[0m        \u001b[1;36m0.064099\u001b[0m   \n",
       "\u001b[1;36m4053\u001b[0m                      Snowc Trend Fall    \u001b[1;36m0.073745\u001b[0m        \u001b[1;36m0.043125\u001b[0m   \n",
       "\n",
       "                                     experiment  grid_level model_type  \\\n",
       "\u001b[1;36m0\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m    XGBoost   \n",
       "\u001b[1;36m1\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m    XGBoost   \n",
       "\u001b[1;36m2\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m    XGBoost   \n",
       "\u001b[1;36m3\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m    XGBoost   \n",
       "\u001b[1;36m4\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m    XGBoost   \n",
       "\u001b[33m...\u001b[0m                                         \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m4049\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m    XGBoost   \n",
       "\u001b[1;36m4050\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m    XGBoost   \n",
       "\u001b[1;36m4051\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m    XGBoost   \n",
       "\u001b[1;36m4052\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m    XGBoost   \n",
       "\u001b[1;36m4053\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m    XGBoost   \n",
       "\n",
       "        task  explanation_number  importance_stability  \n",
       "\u001b[1;36m0\u001b[0m     binary                   \u001b[1;36m2\u001b[0m              \u001b[1;36m1.154834\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m     binary                   \u001b[1;36m2\u001b[0m              \u001b[1;36m3.061753\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m     binary                   \u001b[1;36m2\u001b[0m              \u001b[1;36m1.588574\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m     binary                   \u001b[1;36m2\u001b[0m              \u001b[1;36m1.980514\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m     binary                   \u001b[1;36m2\u001b[0m              \u001b[1;36m0.855361\u001b[0m  \n",
       "\u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m                 \u001b[33m...\u001b[0m                   \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m4049\u001b[0m  binary                 \u001b[1;36m147\u001b[0m              \u001b[1;36m0.731204\u001b[0m  \n",
       "\u001b[1;36m4050\u001b[0m  binary                 \u001b[1;36m147\u001b[0m              \u001b[1;36m1.354661\u001b[0m  \n",
       "\u001b[1;36m4051\u001b[0m  binary                 \u001b[1;36m147\u001b[0m              \u001b[1;36m0.929308\u001b[0m  \n",
       "\u001b[1;36m4052\u001b[0m  binary                 \u001b[1;36m147\u001b[0m              \u001b[1;36m1.111485\u001b[0m  \n",
       "\u001b[1;36m4053\u001b[0m  binary                 \u001b[1;36m147\u001b[0m              \u001b[1;36m0.584792\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m2027\u001b[0m rows x \u001b[1;36m9\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shap_importances_env = shap_importances[shap_importances[\"experiment\"] != \"AlphaEarth\"]\n",
    "shap_importances_env_binary = shap_importances_env[shap_importances_env[\"task\"] == \"binary\"]\n",
    "shap_importances_env_binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "602bbdd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>season</th>\n",
       "      <th>avg_rank</th>\n",
       "      <th>normed_avg_rank</th>\n",
       "      <th>n_features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Winter</td>\n",
       "      <td>99</td>\n",
       "      <td>0.553212</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Spring</td>\n",
       "      <td>93</td>\n",
       "      <td>0.519832</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Summer</td>\n",
       "      <td>80</td>\n",
       "      <td>0.449441</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Fall</td>\n",
       "      <td>90</td>\n",
       "      <td>0.501955</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Winter</td>\n",
       "      <td>92</td>\n",
       "      <td>0.516201</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Spring</td>\n",
       "      <td>86</td>\n",
       "      <td>0.481285</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Summer</td>\n",
       "      <td>97</td>\n",
       "      <td>0.542598</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Fall</td>\n",
       "      <td>98</td>\n",
       "      <td>0.549441</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Winter</td>\n",
       "      <td>102</td>\n",
       "      <td>0.572626</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Spring</td>\n",
       "      <td>89</td>\n",
       "      <td>0.496229</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Summer</td>\n",
       "      <td>82</td>\n",
       "      <td>0.460056</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Fall</td>\n",
       "      <td>92</td>\n",
       "      <td>0.514106</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Winter</td>\n",
       "      <td>89</td>\n",
       "      <td>0.498743</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Spring</td>\n",
       "      <td>95</td>\n",
       "      <td>0.528631</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Summer</td>\n",
       "      <td>91</td>\n",
       "      <td>0.507123</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Fall</td>\n",
       "      <td>79</td>\n",
       "      <td>0.440922</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Winter</td>\n",
       "      <td>21</td>\n",
       "      <td>0.442080</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Spring</td>\n",
       "      <td>25</td>\n",
       "      <td>0.528046</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Summer</td>\n",
       "      <td>26</td>\n",
       "      <td>0.544917</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Fall</td>\n",
       "      <td>19</td>\n",
       "      <td>0.398176</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Winter</td>\n",
       "      <td>26</td>\n",
       "      <td>0.557920</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Spring</td>\n",
       "      <td>21</td>\n",
       "      <td>0.437137</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Summer</td>\n",
       "      <td>27</td>\n",
       "      <td>0.565012</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Fall</td>\n",
       "      <td>19</td>\n",
       "      <td>0.395137</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Winter</td>\n",
       "      <td>23</td>\n",
       "      <td>0.479905</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Spring</td>\n",
       "      <td>22</td>\n",
       "      <td>0.470019</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Summer</td>\n",
       "      <td>25</td>\n",
       "      <td>0.523641</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Fall</td>\n",
       "      <td>21</td>\n",
       "      <td>0.452888</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Winter</td>\n",
       "      <td>22</td>\n",
       "      <td>0.470449</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Spring</td>\n",
       "      <td>30</td>\n",
       "      <td>0.638298</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Summer</td>\n",
       "      <td>24</td>\n",
       "      <td>0.505910</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Fall</td>\n",
       "      <td>12</td>\n",
       "      <td>0.264438</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Winter</td>\n",
       "      <td>26</td>\n",
       "      <td>0.448622</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Spring</td>\n",
       "      <td>29</td>\n",
       "      <td>0.510965</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Summer</td>\n",
       "      <td>27</td>\n",
       "      <td>0.467836</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Fall</td>\n",
       "      <td>31</td>\n",
       "      <td>0.542105</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Winter</td>\n",
       "      <td>28</td>\n",
       "      <td>0.483709</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Spring</td>\n",
       "      <td>30</td>\n",
       "      <td>0.532895</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Summer</td>\n",
       "      <td>27</td>\n",
       "      <td>0.473684</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Fall</td>\n",
       "      <td>34</td>\n",
       "      <td>0.596491</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Winter</td>\n",
       "      <td>26</td>\n",
       "      <td>0.449875</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Spring</td>\n",
       "      <td>30</td>\n",
       "      <td>0.519737</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Summer</td>\n",
       "      <td>31</td>\n",
       "      <td>0.536062</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>Fall</td>\n",
       "      <td>23</td>\n",
       "      <td>0.410526</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Winter</td>\n",
       "      <td>29</td>\n",
       "      <td>0.510025</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Spring</td>\n",
       "      <td>30</td>\n",
       "      <td>0.529605</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Summer</td>\n",
       "      <td>28</td>\n",
       "      <td>0.493177</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Fall</td>\n",
       "      <td>15</td>\n",
       "      <td>0.268421</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                      experiment  grid_level  season  \\\n",
       "\u001b[1;36m0\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m  Winter   \n",
       "\u001b[1;36m1\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m  Spring   \n",
       "\u001b[1;36m2\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m  Summer   \n",
       "\u001b[1;36m3\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m    Fall   \n",
       "\u001b[1;36m4\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m6\u001b[0m  Winter   \n",
       "\u001b[1;36m5\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m6\u001b[0m  Spring   \n",
       "\u001b[1;36m6\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m6\u001b[0m  Summer   \n",
       "\u001b[1;36m7\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m6\u001b[0m    Fall   \n",
       "\u001b[1;36m8\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m4\u001b[0m  Winter   \n",
       "\u001b[1;36m9\u001b[0m                          ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m4\u001b[0m  Spring   \n",
       "\u001b[1;36m10\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m4\u001b[0m  Summer   \n",
       "\u001b[1;36m11\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m4\u001b[0m    Fall   \n",
       "\u001b[1;36m12\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m  Winter   \n",
       "\u001b[1;36m13\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m  Spring   \n",
       "\u001b[1;36m14\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m  Summer   \n",
       "\u001b[1;36m15\u001b[0m                         ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m    Fall   \n",
       "\u001b[1;36m16\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m  Winter   \n",
       "\u001b[1;36m17\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m  Spring   \n",
       "\u001b[1;36m18\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m  Summer   \n",
       "\u001b[1;36m19\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m    Fall   \n",
       "\u001b[1;36m20\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  Winter   \n",
       "\u001b[1;36m21\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  Spring   \n",
       "\u001b[1;36m22\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  Summer   \n",
       "\u001b[1;36m23\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m    Fall   \n",
       "\u001b[1;36m24\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m  Winter   \n",
       "\u001b[1;36m25\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m  Spring   \n",
       "\u001b[1;36m26\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m  Summer   \n",
       "\u001b[1;36m27\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m    Fall   \n",
       "\u001b[1;36m28\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m  Winter   \n",
       "\u001b[1;36m29\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m  Spring   \n",
       "\u001b[1;36m30\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m  Summer   \n",
       "\u001b[1;36m31\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m    Fall   \n",
       "\u001b[1;36m32\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m  Winter   \n",
       "\u001b[1;36m33\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m  Spring   \n",
       "\u001b[1;36m34\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m  Summer   \n",
       "\u001b[1;36m35\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m    Fall   \n",
       "\u001b[1;36m36\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  Winter   \n",
       "\u001b[1;36m37\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  Spring   \n",
       "\u001b[1;36m38\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  Summer   \n",
       "\u001b[1;36m39\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m    Fall   \n",
       "\u001b[1;36m40\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m  Winter   \n",
       "\u001b[1;36m41\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m  Spring   \n",
       "\u001b[1;36m42\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m  Summer   \n",
       "\u001b[1;36m43\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m    Fall   \n",
       "\u001b[1;36m44\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m  Winter   \n",
       "\u001b[1;36m45\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m  Spring   \n",
       "\u001b[1;36m46\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m  Summer   \n",
       "\u001b[1;36m47\u001b[0m     ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m    Fall   \n",
       "\n",
       "    avg_rank  normed_avg_rank  n_features  \n",
       "\u001b[1;36m0\u001b[0m         \u001b[1;36m99\u001b[0m         \u001b[1;36m0.553212\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m         \u001b[1;36m93\u001b[0m         \u001b[1;36m0.519832\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m         \u001b[1;36m80\u001b[0m         \u001b[1;36m0.449441\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m         \u001b[1;36m90\u001b[0m         \u001b[1;36m0.501955\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m         \u001b[1;36m92\u001b[0m         \u001b[1;36m0.516201\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m5\u001b[0m         \u001b[1;36m86\u001b[0m         \u001b[1;36m0.481285\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m6\u001b[0m         \u001b[1;36m97\u001b[0m         \u001b[1;36m0.542598\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m7\u001b[0m         \u001b[1;36m98\u001b[0m         \u001b[1;36m0.549441\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m8\u001b[0m        \u001b[1;36m102\u001b[0m         \u001b[1;36m0.572626\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m9\u001b[0m         \u001b[1;36m89\u001b[0m         \u001b[1;36m0.496229\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m10\u001b[0m        \u001b[1;36m82\u001b[0m         \u001b[1;36m0.460056\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m11\u001b[0m        \u001b[1;36m92\u001b[0m         \u001b[1;36m0.514106\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m12\u001b[0m        \u001b[1;36m89\u001b[0m         \u001b[1;36m0.498743\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m13\u001b[0m        \u001b[1;36m95\u001b[0m         \u001b[1;36m0.528631\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m14\u001b[0m        \u001b[1;36m91\u001b[0m         \u001b[1;36m0.507123\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m15\u001b[0m        \u001b[1;36m79\u001b[0m         \u001b[1;36m0.440922\u001b[0m        \u001b[1;36m40.0\u001b[0m  \n",
       "\u001b[1;36m16\u001b[0m        \u001b[1;36m21\u001b[0m         \u001b[1;36m0.442080\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m17\u001b[0m        \u001b[1;36m25\u001b[0m         \u001b[1;36m0.528046\u001b[0m        \u001b[1;36m11.0\u001b[0m  \n",
       "\u001b[1;36m18\u001b[0m        \u001b[1;36m26\u001b[0m         \u001b[1;36m0.544917\u001b[0m        \u001b[1;36m18.0\u001b[0m  \n",
       "\u001b[1;36m19\u001b[0m        \u001b[1;36m19\u001b[0m         \u001b[1;36m0.398176\u001b[0m         \u001b[1;36m7.0\u001b[0m  \n",
       "\u001b[1;36m20\u001b[0m        \u001b[1;36m26\u001b[0m         \u001b[1;36m0.557920\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m21\u001b[0m        \u001b[1;36m21\u001b[0m         \u001b[1;36m0.437137\u001b[0m        \u001b[1;36m11.0\u001b[0m  \n",
       "\u001b[1;36m22\u001b[0m        \u001b[1;36m27\u001b[0m         \u001b[1;36m0.565012\u001b[0m        \u001b[1;36m18.0\u001b[0m  \n",
       "\u001b[1;36m23\u001b[0m        \u001b[1;36m19\u001b[0m         \u001b[1;36m0.395137\u001b[0m         \u001b[1;36m7.0\u001b[0m  \n",
       "\u001b[1;36m24\u001b[0m        \u001b[1;36m23\u001b[0m         \u001b[1;36m0.479905\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m25\u001b[0m        \u001b[1;36m22\u001b[0m         \u001b[1;36m0.470019\u001b[0m        \u001b[1;36m11.0\u001b[0m  \n",
       "\u001b[1;36m26\u001b[0m        \u001b[1;36m25\u001b[0m         \u001b[1;36m0.523641\u001b[0m        \u001b[1;36m18.0\u001b[0m  \n",
       "\u001b[1;36m27\u001b[0m        \u001b[1;36m21\u001b[0m         \u001b[1;36m0.452888\u001b[0m         \u001b[1;36m7.0\u001b[0m  \n",
       "\u001b[1;36m28\u001b[0m        \u001b[1;36m22\u001b[0m         \u001b[1;36m0.470449\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m29\u001b[0m        \u001b[1;36m30\u001b[0m         \u001b[1;36m0.638298\u001b[0m        \u001b[1;36m11.0\u001b[0m  \n",
       "\u001b[1;36m30\u001b[0m        \u001b[1;36m24\u001b[0m         \u001b[1;36m0.505910\u001b[0m        \u001b[1;36m18.0\u001b[0m  \n",
       "\u001b[1;36m31\u001b[0m        \u001b[1;36m12\u001b[0m         \u001b[1;36m0.264438\u001b[0m         \u001b[1;36m7.0\u001b[0m  \n",
       "\u001b[1;36m32\u001b[0m        \u001b[1;36m26\u001b[0m         \u001b[1;36m0.448622\u001b[0m        \u001b[1;36m14.0\u001b[0m  \n",
       "\u001b[1;36m33\u001b[0m        \u001b[1;36m29\u001b[0m         \u001b[1;36m0.510965\u001b[0m        \u001b[1;36m16.0\u001b[0m  \n",
       "\u001b[1;36m34\u001b[0m        \u001b[1;36m27\u001b[0m         \u001b[1;36m0.467836\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m35\u001b[0m        \u001b[1;36m31\u001b[0m         \u001b[1;36m0.542105\u001b[0m        \u001b[1;36m10.0\u001b[0m  \n",
       "\u001b[1;36m36\u001b[0m        \u001b[1;36m28\u001b[0m         \u001b[1;36m0.483709\u001b[0m        \u001b[1;36m14.0\u001b[0m  \n",
       "\u001b[1;36m37\u001b[0m        \u001b[1;36m30\u001b[0m         \u001b[1;36m0.532895\u001b[0m        \u001b[1;36m16.0\u001b[0m  \n",
       "\u001b[1;36m38\u001b[0m        \u001b[1;36m27\u001b[0m         \u001b[1;36m0.473684\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m39\u001b[0m        \u001b[1;36m34\u001b[0m         \u001b[1;36m0.596491\u001b[0m        \u001b[1;36m10.0\u001b[0m  \n",
       "\u001b[1;36m40\u001b[0m        \u001b[1;36m26\u001b[0m         \u001b[1;36m0.449875\u001b[0m        \u001b[1;36m14.0\u001b[0m  \n",
       "\u001b[1;36m41\u001b[0m        \u001b[1;36m30\u001b[0m         \u001b[1;36m0.519737\u001b[0m        \u001b[1;36m16.0\u001b[0m  \n",
       "\u001b[1;36m42\u001b[0m        \u001b[1;36m31\u001b[0m         \u001b[1;36m0.536062\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m43\u001b[0m        \u001b[1;36m23\u001b[0m         \u001b[1;36m0.410526\u001b[0m        \u001b[1;36m10.0\u001b[0m  \n",
       "\u001b[1;36m44\u001b[0m        \u001b[1;36m29\u001b[0m         \u001b[1;36m0.510025\u001b[0m        \u001b[1;36m14.0\u001b[0m  \n",
       "\u001b[1;36m45\u001b[0m        \u001b[1;36m30\u001b[0m         \u001b[1;36m0.529605\u001b[0m        \u001b[1;36m16.0\u001b[0m  \n",
       "\u001b[1;36m46\u001b[0m        \u001b[1;36m28\u001b[0m         \u001b[1;36m0.493177\u001b[0m         \u001b[1;36m9.0\u001b[0m  \n",
       "\u001b[1;36m47\u001b[0m        \u001b[1;36m15\u001b[0m         \u001b[1;36m0.268421\u001b[0m        \u001b[1;36m10.0\u001b[0m  "
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_season_ranks = []\n",
    "seasons = [\"Winter\", \"Spring\", \"Summer\", \"Fall\"]\n",
    "for experiment in shap_importances_env_binary[\"experiment\"].unique():\n",
    "    for grid_level in shap_importances_env_binary[\"grid_level\"].unique():\n",
    "        if grid_level.startswith(\"HEALPix\"):\n",
    "            continue\n",
    "        subset = (\n",
    "            shap_importances_env_binary[\n",
    "                (shap_importances_env_binary[\"experiment\"] == experiment)\n",
    "                & (shap_importances_env_binary[\"grid_level\"] == grid_level)\n",
    "            ]\n",
    "            .sort_values(\"importance\", ascending=False)\n",
    "            .reset_index(drop=True)\n",
    "            .reset_index(names=\"rank\")\n",
    "        )\n",
    "        for season in seasons:\n",
    "            season_subset = subset[subset[\"feature\"].str.contains(season)]\n",
    "            avg_rank = int(season_subset[\"rank\"].mean().round(0))\n",
    "            normed_avg_rank = season_subset[\"rank\"].mean() / len(subset)\n",
    "            avg_season_ranks.append(\n",
    "                {\n",
    "                    \"experiment\": experiment,\n",
    "                    \"grid_level\": grid_level,\n",
    "                    \"season\": season,\n",
    "                    \"avg_rank\": avg_rank,\n",
    "                    \"normed_avg_rank\": normed_avg_rank,\n",
    "                    \"n_features\": float(len(season_subset)),\n",
    "                }\n",
    "            )\n",
    "avg_season_ranks = pd.DataFrame(avg_season_ranks)\n",
    "avg_season_ranks[\"grid_level\"] = avg_season_ranks[\"grid_level\"].astype(grid_level_cat).cat.remove_unused_categories()\n",
    "avg_season_ranks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19026eda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 169748 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn._core.plot.Plot\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f3aa3876ed0\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 188,
     "metadata": {
      "image/png": {
       "height": 368.47499999999997,
       "width": 1332.8
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = (\n",
    "    so.Plot(avg_season_ranks, x=\"season\", y=\"normed_avg_rank\", color=\"grid_level\")\n",
    "    .add(so.Bar(), so.Dodge())\n",
    "    .facet(\"experiment\")\n",
    "    .layout(size=(15, 4.5), engine=\"constrained\")\n",
    "    .scale(color=\"crest\")\n",
    "    .label(\n",
    "        x=\"Season\",\n",
    "        y=\"Average SHAP Importance Rank (Normalized)\",\n",
    "        color=\"Grid Level\",\n",
    "    )\n",
    "    .theme(plt.rcParams)\n",
    ")\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "97ac84a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>variable</th>\n",
       "      <th>avg_rank</th>\n",
       "      <th>normed_avg_rank</th>\n",
       "      <th>std_rank</th>\n",
       "      <th>normed_std_rank</th>\n",
       "      <th>n_features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>T2M</td>\n",
       "      <td>71</td>\n",
       "      <td>0.397579</td>\n",
       "      <td>49.862079</td>\n",
       "      <td>0.278559</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Tp</td>\n",
       "      <td>93</td>\n",
       "      <td>0.522092</td>\n",
       "      <td>41.690199</td>\n",
       "      <td>0.232906</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Effective Snow Depth</td>\n",
       "      <td>93</td>\n",
       "      <td>0.520950</td>\n",
       "      <td>49.189865</td>\n",
       "      <td>0.274804</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Freezing Days</td>\n",
       "      <td>121</td>\n",
       "      <td>0.679469</td>\n",
       "      <td>58.287312</td>\n",
       "      <td>0.325627</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Freezing Degree Days</td>\n",
       "      <td>55</td>\n",
       "      <td>0.312151</td>\n",
       "      <td>48.398458</td>\n",
       "      <td>0.270382</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Precipitation Occurrences</td>\n",
       "      <td>25</td>\n",
       "      <td>0.441520</td>\n",
       "      <td>14.985549</td>\n",
       "      <td>0.262904</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Lblt</td>\n",
       "      <td>34</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>8.497058</td>\n",
       "      <td>0.149071</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Sf</td>\n",
       "      <td>42</td>\n",
       "      <td>0.748538</td>\n",
       "      <td>9.291573</td>\n",
       "      <td>0.163010</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Snowc</td>\n",
       "      <td>27</td>\n",
       "      <td>0.485380</td>\n",
       "      <td>22.810816</td>\n",
       "      <td>0.400190</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Sshf</td>\n",
       "      <td>26</td>\n",
       "      <td>0.464912</td>\n",
       "      <td>19.122413</td>\n",
       "      <td>0.335481</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                    experiment  grid_level  \\\n",
       "\u001b[1;36m0\u001b[0m                        ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                        ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m                        ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                        ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                        ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "..                                         \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m135\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m136\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m137\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m138\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m139\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\n",
       "                      variable  avg_rank  normed_avg_rank   std_rank  \\\n",
       "\u001b[1;36m0\u001b[0m                          T2M        \u001b[1;36m71\u001b[0m         \u001b[1;36m0.397579\u001b[0m  \u001b[1;36m49.862079\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                           Tp        \u001b[1;36m93\u001b[0m         \u001b[1;36m0.522092\u001b[0m  \u001b[1;36m41.690199\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m         Effective Snow Depth        \u001b[1;36m93\u001b[0m         \u001b[1;36m0.520950\u001b[0m  \u001b[1;36m49.189865\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                Freezing Days       \u001b[1;36m121\u001b[0m         \u001b[1;36m0.679469\u001b[0m  \u001b[1;36m58.287312\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m         Freezing Degree Days        \u001b[1;36m55\u001b[0m         \u001b[1;36m0.312151\u001b[0m  \u001b[1;36m48.398458\u001b[0m   \n",
       "..                         \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m              \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m135\u001b[0m  Precipitation Occurrences        \u001b[1;36m25\u001b[0m         \u001b[1;36m0.441520\u001b[0m  \u001b[1;36m14.985549\u001b[0m   \n",
       "\u001b[1;36m136\u001b[0m                       Lblt        \u001b[1;36m34\u001b[0m         \u001b[1;36m0.600000\u001b[0m   \u001b[1;36m8.497058\u001b[0m   \n",
       "\u001b[1;36m137\u001b[0m                         Sf        \u001b[1;36m42\u001b[0m         \u001b[1;36m0.748538\u001b[0m   \u001b[1;36m9.291573\u001b[0m   \n",
       "\u001b[1;36m138\u001b[0m                      Snowc        \u001b[1;36m27\u001b[0m         \u001b[1;36m0.485380\u001b[0m  \u001b[1;36m22.810816\u001b[0m   \n",
       "\u001b[1;36m139\u001b[0m                       Sshf        \u001b[1;36m26\u001b[0m         \u001b[1;36m0.464912\u001b[0m  \u001b[1;36m19.122413\u001b[0m   \n",
       "\n",
       "     normed_std_rank  n_features  \n",
       "\u001b[1;36m0\u001b[0m           \u001b[1;36m0.278559\u001b[0m        \u001b[1;36m48.0\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m           \u001b[1;36m0.232906\u001b[0m        \u001b[1;36m11.0\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m           \u001b[1;36m0.274804\u001b[0m         \u001b[1;36m8.0\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m           \u001b[1;36m0.325627\u001b[0m         \u001b[1;36m8.0\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m           \u001b[1;36m0.270382\u001b[0m         \u001b[1;36m8.0\u001b[0m  \n",
       "..               \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m135\u001b[0m         \u001b[1;36m0.262904\u001b[0m         \u001b[1;36m6.0\u001b[0m  \n",
       "\u001b[1;36m136\u001b[0m         \u001b[1;36m0.149071\u001b[0m         \u001b[1;36m5.0\u001b[0m  \n",
       "\u001b[1;36m137\u001b[0m         \u001b[1;36m0.163010\u001b[0m         \u001b[1;36m3.0\u001b[0m  \n",
       "\u001b[1;36m138\u001b[0m         \u001b[1;36m0.400190\u001b[0m         \u001b[1;36m3.0\u001b[0m  \n",
       "\u001b[1;36m139\u001b[0m         \u001b[1;36m0.335481\u001b[0m         \u001b[1;36m4.0\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m140\u001b[0m rows x \u001b[1;36m8\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_variable_ranks = []\n",
    "variables = [\n",
    "    \"T2M\",\n",
    "    \"Tp\",\n",
    "    \"Effective Snow Depth\",\n",
    "    \"Freezing Days\",\n",
    "    \"Freezing Degree Days\",\n",
    "    \"Thawing Days\",\n",
    "    \"Thawing Degree Days\",\n",
    "    \"Precipitation Occurrences\",\n",
    "    \"Snowfall Occurrences\",\n",
    "    \"Lblt\",\n",
    "    \"Naive Snow Isolation\",\n",
    "    \"Sde\",\n",
    "    \"Sf\",\n",
    "    \"Snowc\",\n",
    "    \"Sshf\",\n",
    "]\n",
    "for experiment in shap_importances_env_binary[\"experiment\"].unique():\n",
    "    for grid_level in shap_importances_env_binary[\"grid_level\"].unique():\n",
    "        if grid_level.startswith(\"HEALPix\"):\n",
    "            continue\n",
    "        subset = (\n",
    "            shap_importances_env_binary[\n",
    "                (shap_importances_env_binary[\"experiment\"] == experiment)\n",
    "                & (shap_importances_env_binary[\"grid_level\"] == grid_level)\n",
    "            ]\n",
    "            .sort_values(\"importance\", ascending=False)\n",
    "            .reset_index(drop=True)\n",
    "            .reset_index(names=\"rank\")\n",
    "        )\n",
    "        for v in variables:\n",
    "            variable_subset = subset[subset[\"feature\"].str.contains(v)]\n",
    "            if len(variable_subset) == 0:\n",
    "                continue\n",
    "            avg_rank = int(variable_subset[\"rank\"].mean())\n",
    "            normed_avg_rank = variable_subset[\"rank\"].mean() / len(subset)\n",
    "            std_rank = variable_subset[\"rank\"].std()\n",
    "            normed_std_rank = std_rank / len(subset)\n",
    "            avg_variable_ranks.append(\n",
    "                {\n",
    "                    \"experiment\": experiment,\n",
    "                    \"grid_level\": grid_level,\n",
    "                    \"variable\": v,\n",
    "                    \"avg_rank\": avg_rank,\n",
    "                    \"normed_avg_rank\": normed_avg_rank,\n",
    "                    \"std_rank\": std_rank,\n",
    "                    \"normed_std_rank\": normed_std_rank,\n",
    "                    \"n_features\": float(len(variable_subset)),\n",
    "                }\n",
    "            )\n",
    "avg_variable_ranks = pd.DataFrame(avg_variable_ranks)\n",
    "avg_variable_ranks[\"grid_level\"] = (\n",
    "    avg_variable_ranks[\"grid_level\"].astype(grid_level_cat).cat.remove_unused_categories()\n",
    ")\n",
    "avg_variable_ranks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65c95cd2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 175404 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 150\u001b[0m\u001b[1;36m0x800\u001b[0m\u001b[39m with \u001b[0m\u001b[1;36m3\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = (\n",
    "    so.Plot(avg_variable_ranks, x=\"variable\", y=\"normed_avg_rank\", color=\"grid_level\")\n",
    "    .add(so.Bar(), so.Dodge())\n",
    "    .facet(row=\"experiment\")\n",
    "    .layout(size=(15, 8), engine=\"constrained\")\n",
    "    .scale(color=\"crest\")\n",
    "    .label(\n",
    "        x=\"Variable\",\n",
    "        y=\"Average SHAP Importance Rank (Normalized)\",\n",
    "        color=\"Grid Level\",\n",
    "    )\n",
    "    .theme(plt.rcParams)\n",
    ")\n",
    "fig = p.plot()._figure\n",
    "\n",
    "for i, ax in enumerate(fig.axes):\n",
    "    for label in ax.get_xticklabels():\n",
    "        # Align x-axis labels right and rotate them by 45 degrees\n",
    "        label.set_horizontalalignment(\"right\")\n",
    "        label.set_rotation(45)\n",
    "    if i != 1:\n",
    "        # Disable y axis labels for all but the middle plot\n",
    "        ax.set_ylabel(\"\")\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89235a88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot all shap importances per features\n",
    "p = (\n",
    "    so.Plot(shap_importances, y=\"feature\", x=\"importance\")\n",
    "    .add(so.Bar())\n",
    "    .label(x=\"SHAP Importance\", y=\"Feature\")\n",
    "    .layout(size=(15, 8))\n",
    ")\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5a503643",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2136736/537463495.py:4: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  .groupby([\"experiment\", \"grid_level\"])\n",
      "/tmp/ipykernel_2136736/537463495.py:5: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  .apply(lambda df: df.nlargest(10, \"importance\"))\n",
      "/tmp/ipykernel_2136736/537463495.py:7: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  .groupby([\"experiment\", \"grid_level\"])[\"feature\"]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>feature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>{Embeddings A58, Y, Embeddings A16, Embeddings...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>{Embeddings A60, Cell Area, Embeddings A40, Y,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>{Cell Area, Embeddings A60, Embeddings A40, Em...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>{Cell Area, Embeddings A60, Embeddings A59, Em...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>{T2M Spring, T2M Avg Trend Fall, T2M Avg Trend...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>{Freezing Degree Days Trend Fall, T2M Min Tren...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>{T2M Skew Winter, T2M Avg Winter, Thawing Days...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>{Freezing Degree Days Trend Summer, Thawing Da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>{Freezing Days Summer, Freezing Degree Days Tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>{Freezing Days Summer, Freezing Degree Days Tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>{Freezing Days Summer, Freezing Degree Days Tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>{Freezing Days Summer, Freezing Degree Days Tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>{Freezing Degree Days Trend Fall, T2M Avg Tren...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>{T2M Avg Winter, Freezing Degree Days Trend Su...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>{T2M Max Winter, T2M Avg Winter, Freezing Degr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>{Y, T2M Max Fall, Freezing Days Trend Summer, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                 experiment  grid_level  \\\n",
       "\u001b[1;36m0\u001b[0m                                AlphaEarth  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                                AlphaEarth  H3 Level \u001b[1;36m4\u001b[0m   \n",
       "\u001b[1;36m5\u001b[0m                                AlphaEarth  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m8\u001b[0m                                AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m   \n",
       "\u001b[1;36m9\u001b[0m                          ERA5 + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m12\u001b[0m                         ERA5 + ArcticDEM  H3 Level \u001b[1;36m4\u001b[0m   \n",
       "\u001b[1;36m14\u001b[0m                         ERA5 + ArcticDEM  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m17\u001b[0m                         ERA5 + ArcticDEM  H3 Level \u001b[1;36m6\u001b[0m   \n",
       "\u001b[1;36m18\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m21\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m   \n",
       "\u001b[1;36m23\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m26\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m   \n",
       "\u001b[1;36m27\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m3\u001b[0m   \n",
       "\u001b[1;36m30\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m4\u001b[0m   \n",
       "\u001b[1;36m32\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m5\u001b[0m   \n",
       "\u001b[1;36m35\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m   \n",
       "\n",
       "                                              feature  \n",
       "\u001b[1;36m0\u001b[0m   \u001b[1m{\u001b[0mEmbeddings A58, Y, Embeddings A16, Embeddings\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m   \u001b[1m{\u001b[0mEmbeddings A60, Cell Area, Embeddings A40, Y,\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m5\u001b[0m   \u001b[1m{\u001b[0mCell Area, Embeddings A60, Embeddings A40, Em\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m8\u001b[0m   \u001b[1m{\u001b[0mCell Area, Embeddings A60, Embeddings A59, Em\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m9\u001b[0m   \u001b[1m{\u001b[0mT2M Spring, T2M Avg Trend Fall, T2M Avg Trend\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m12\u001b[0m  \u001b[1m{\u001b[0mFreezing Degree Days Trend Fall, T2M Min Tren\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m14\u001b[0m  \u001b[1m{\u001b[0mT2M Skew Winter, T2M Avg Winter, Thawing Days\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m17\u001b[0m  \u001b[1m{\u001b[0mFreezing Degree Days Trend Summer, Thawing Da\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m18\u001b[0m  \u001b[1m{\u001b[0mFreezing Days Summer, Freezing Degree Days Tr\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m21\u001b[0m  \u001b[1m{\u001b[0mFreezing Days Summer, Freezing Degree Days Tr\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m23\u001b[0m  \u001b[1m{\u001b[0mFreezing Days Summer, Freezing Degree Days Tr\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m26\u001b[0m  \u001b[1m{\u001b[0mFreezing Days Summer, Freezing Degree Days Tr\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m27\u001b[0m  \u001b[1m{\u001b[0mFreezing Degree Days Trend Fall, T2M Avg Tren\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m30\u001b[0m  \u001b[1m{\u001b[0mT2M Avg Winter, Freezing Degree Days Trend Su\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m32\u001b[0m  \u001b[1m{\u001b[0mT2M Max Winter, T2M Avg Winter, Freezing Degr\u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m35\u001b[0m  \u001b[1m{\u001b[0mY, T2M Max Fall, Freezing Days Trend Summer, \u001b[33m...\u001b[0m  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the top 10 most important features for each experiment and grid level\n",
    "top_features = (\n",
    "    shap_importances[shap_importances[\"grid_level\"].str.startswith(\"H3\")]\n",
    "    .groupby([\"experiment\", \"grid_level\"])\n",
    "    .apply(lambda df: df.nlargest(10, \"importance\"))\n",
    "    .reset_index(drop=True)\n",
    "    .groupby([\"experiment\", \"grid_level\"])[\"feature\"]\n",
    "    .apply(set)\n",
    "    .reset_index()\n",
    "    .dropna()\n",
    ")\n",
    "top_features[\"grid_level\"] = top_features[\"grid_level\"].cat.remove_unused_categories()\n",
    "top_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "37942cae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>feature</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Embeddings A16</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Embeddings A49</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Embeddings A50</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>X</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Y</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>T2M Max Winter</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>T2M Winter</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>Thawing Degree Days Spring</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>X</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>Y</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                 experiment                     feature  count\n",
       "\u001b[1;36m2\u001b[0m                                AlphaEarth              Embeddings A16      \u001b[1;36m4\u001b[0m\n",
       "\u001b[1;36m9\u001b[0m                                AlphaEarth              Embeddings A49      \u001b[1;36m4\u001b[0m\n",
       "\u001b[1;36m10\u001b[0m                               AlphaEarth              Embeddings A50      \u001b[1;36m4\u001b[0m\n",
       "\u001b[1;36m15\u001b[0m                               AlphaEarth                           X      \u001b[1;36m4\u001b[0m\n",
       "\u001b[1;36m16\u001b[0m                               AlphaEarth                           Y      \u001b[1;36m4\u001b[0m\n",
       "..                                      \u001b[33m...\u001b[0m                         \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m\n",
       "\u001b[1;36m66\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m              T2M Max Winter      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m69\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m                  T2M Winter      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m71\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  Thawing Degree Days Spring      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m73\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m                           X      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m74\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m                           Y      \u001b[1;36m1\u001b[0m\n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m75\u001b[0m rows x \u001b[1;36m3\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count of many times a feature appears in the top 10 across all grid levels, keeping the experiment separate\n",
    "top_features_count_per_experiment = (\n",
    "    top_features.explode(\"feature\").groupby([\"experiment\", \"feature\"]).size().reset_index(name=\"count\")\n",
    ")\n",
    "top_features_count_per_experiment.sort_values([\"experiment\", \"count\"], ascending=[True, False], inplace=True)\n",
    "top_features_count_per_experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "6c10b092",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 644300 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn._core.plot.Plot\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f6caa73bc50\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "image/png": {
       "height": 639.625,
       "width": 1210.825
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Plot this per experiment\n",
    "p = (\n",
    "    so.Plot(top_features_count_per_experiment, y=\"feature\", x=\"count\")\n",
    "    .facet(\"experiment\", wrap=2)\n",
    "    .share(x=True, y=False)\n",
    "    .add(so.Bar())\n",
    "    .label(x=\"Feature\", y=\"Count in Top 10\")\n",
    "    .layout(size=(15, 8))\n",
    ")\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57b54f87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2136736/2791643824.py:3: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  top_features[top_features[\"experiment\"] != \"AlphaEarth\"].explode(\"feature\").groupby([\"grid_level\", \"feature\"]).size().reset_index(name=\"count\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>feature</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Freezing Days Trend Summer</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Freezing Degree Days Trend Fall</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>Precipitation Occurrences Spring</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>T2M Max Trend Fall</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>T2M Summer</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>T2M Max Fall</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>T2M Min Spring</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>T2M Skew Winter</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Thawing Days Trend Summer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>Thawing Degree Days Spring</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "     grid_level                           feature  count\n",
       "\u001b[1;36m5\u001b[0m    H3 Level \u001b[1;36m3\u001b[0m        Freezing Days Trend Summer      \u001b[1;36m2\u001b[0m\n",
       "\u001b[1;36m8\u001b[0m    H3 Level \u001b[1;36m3\u001b[0m   Freezing Degree Days Trend Fall      \u001b[1;36m2\u001b[0m\n",
       "\u001b[1;36m12\u001b[0m   H3 Level \u001b[1;36m3\u001b[0m  Precipitation Occurrences Spring      \u001b[1;36m2\u001b[0m\n",
       "\u001b[1;36m26\u001b[0m   H3 Level \u001b[1;36m3\u001b[0m                T2M Max Trend Fall      \u001b[1;36m2\u001b[0m\n",
       "\u001b[1;36m33\u001b[0m   H3 Level \u001b[1;36m3\u001b[0m                        T2M Summer      \u001b[1;36m2\u001b[0m\n",
       "..          \u001b[33m...\u001b[0m                               \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m\n",
       "\u001b[1;36m148\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m                      T2M Max Fall      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m152\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m                    T2M Min Spring      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m154\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m                   T2M Skew Winter      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m159\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m         Thawing Days Trend Summer      \u001b[1;36m1\u001b[0m\n",
       "\u001b[1;36m160\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        Thawing Degree Days Spring      \u001b[1;36m1\u001b[0m\n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m92\u001b[0m rows x \u001b[1;36m3\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count of many times a feature appears in the top 10 across all experiments, keeping the grid_level separate\n",
    "top_features_count_per_grid_level = (\n",
    "    top_features[top_features[\"experiment\"] != \"AlphaEarth\"]\n",
    "    .explode(\"feature\")\n",
    "    .groupby([\"grid_level\", \"feature\"])\n",
    "    .size()\n",
    "    .reset_index(name=\"count\")\n",
    ")\n",
    "top_features_count_per_grid_level.sort_values([\"grid_level\", \"count\"], ascending=[True, False], inplace=True)\n",
    "top_features_count_per_grid_level = top_features_count_per_grid_level[top_features_count_per_grid_level[\"count\"] > 0]\n",
    "top_features_count_per_grid_level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "27124727",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 766388 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn._core.plot.Plot\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f6ca9437050\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 48,
     "metadata": {
      "image/png": {
       "height": 639.625,
       "width": 1210.825
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Plot this per grid_level\n",
    "p = (\n",
    "    so.Plot(top_features_count_per_grid_level, y=\"feature\", x=\"count\")\n",
    "    .facet(\"grid_level\", wrap=2)\n",
    "    .share(x=True, y=False)\n",
    "    .add(so.Bar())\n",
    "    .label(x=\"Feature\", y=\"Count in Top 10\")\n",
    "    .layout(size=(15, 8))\n",
    ")\n",
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9208d9c3",
   "metadata": {},
   "source": [
    "## eSPA Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "53fd698e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading grid took 0.04s\n",
      "Getting target labels (with task=binary) took 0.10s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.26s\n",
      "Getting target labels (with task=binary) took 0.33s\n",
      "Loading features from cache took 0.08s\n",
      "Reading grid took 1.74s\n",
      "Getting target labels (with task=binary) took 1.99s\n",
      "Loading features from cache took 0.13s\n",
      "Reading grid took 0.03s\n",
      "Getting target labels (with task=binary) took 0.05s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.07s\n",
      "Getting target labels (with task=binary) took 0.10s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.06s\n",
      "Getting target labels (with task=binary) took 0.08s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.27s\n",
      "Getting target labels (with task=binary) took 0.29s\n",
      "Loading features from cache took 0.04s\n",
      "Reading grid took 0.90s\n",
      "Getting target labels (with task=binary) took 1.30s\n",
      "Loading features from cache took 0.09s\n",
      "Reading grid took 0.04s\n",
      "Getting target labels (with task=binary) took 0.06s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.25s\n",
      "Getting target labels (with task=binary) took 0.28s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.03s\n",
      "Getting target labels (with task=binary) took 0.05s\n",
      "Loading features from cache took 0.00s\n",
      "Reading grid took 0.06s\n",
      "Getting target labels (with task=binary) took 0.08s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 1.67s\n",
      "Getting target labels (with task=binary) took 1.94s\n",
      "Loading features from cache took 0.04s\n",
      "Reading grid took 0.24s\n",
      "Getting target labels (with task=binary) took 0.26s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.06s\n",
      "Getting target labels (with task=binary) took 0.08s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.03s\n",
      "Getting target labels (with task=binary) took 0.05s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 1.68s\n",
      "Getting target labels (with task=binary) took 1.94s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.04s\n",
      "Getting target labels (with task=binary) took 0.06s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.24s\n",
      "Getting target labels (with task=binary) took 0.31s\n",
      "Loading features from cache took 0.16s\n",
      "Reading grid took 0.06s\n",
      "Getting target labels (with task=binary) took 0.08s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.03s\n",
      "Getting target labels (with task=binary) took 0.05s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 1.75s\n",
      "Getting target labels (with task=binary) took 2.01s\n",
      "Loading features from cache took 0.13s\n",
      "Reading grid took 0.86s\n",
      "Getting target labels (with task=binary) took 1.26s\n",
      "Loading features from cache took 0.07s\n",
      "Reading grid took 0.03s\n",
      "Getting target labels (with task=binary) took 0.05s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.23s\n",
      "Getting target labels (with task=binary) took 0.26s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.08s\n",
      "Getting target labels (with task=binary) took 0.10s\n",
      "Loading features from cache took 0.01s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>feature_weights</th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cell Area</td>\n",
       "      <td>0.005587</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Land Area</td>\n",
       "      <td>0.005587</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Water Area</td>\n",
       "      <td>0.005587</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>0.005587</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>X</td>\n",
       "      <td>0.005587</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3228</th>\n",
       "      <td>Embeddings Trend A59</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3229</th>\n",
       "      <td>Embeddings Trend A60</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>Embeddings Trend A61</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3231</th>\n",
       "      <td>Embeddings Trend A62</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3232</th>\n",
       "      <td>Embeddings Trend A63</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3233 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                   feature  feature_weights             experiment  \\\n",
       "\u001b[1;36m0\u001b[0m                Cell Area         \u001b[1;36m0.005587\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m1\u001b[0m                Land Area         \u001b[1;36m0.005587\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m2\u001b[0m               Water Area         \u001b[1;36m0.005587\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m3\u001b[0m               Land Ratio         \u001b[1;36m0.005587\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m4\u001b[0m                        X         \u001b[1;36m0.005587\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m              \u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m3228\u001b[0m  Embeddings Trend A59         \u001b[1;36m0.007692\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3229\u001b[0m  Embeddings Trend A60         \u001b[1;36m0.007692\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3230\u001b[0m  Embeddings Trend A61         \u001b[1;36m0.007692\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3231\u001b[0m  Embeddings Trend A62         \u001b[1;36m0.007692\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3232\u001b[0m  Embeddings Trend A63         \u001b[1;36m0.007692\u001b[0m             AlphaEarth   \n",
       "\n",
       "           grid_level model_type    task  \n",
       "\u001b[1;36m0\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m1\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m2\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m4\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[33m...\u001b[0m               \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m3228\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3229\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3230\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3231\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3232\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m3233\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from entropice.ml.models import extract_espa_state\n",
    "\n",
    "espa_experiments = all_experiments[all_experiments[\"model_type\"] == \"eSPA\"]\n",
    "weights = []\n",
    "for i, row in espa_experiments.iterrows():\n",
    "    run = row[\"run\"]\n",
    "    task = run.task\n",
    "    grid = run.dataset.grid.replace(\"hex\", \"H3\").replace(\"healpix\", \"HEALPix\")\n",
    "    grid_level = f\"{grid} Level {run.dataset.level}\"\n",
    "    model_type = (\n",
    "        run.model_type.replace(\"rf\", \"Random Forest\")\n",
    "        .replace(\"xgboost\", \"XGBoost\")\n",
    "        .replace(\"knn\", \"kNN\")\n",
    "        .replace(\"espa\", \"eSPA\")\n",
    "        .replace(\"autogluon\", \"AutoGluon\")\n",
    "    )\n",
    "    espa_state = extract_espa_state(run.model.named_steps[\"model\"], run.training_set)\n",
    "    espa_weights = espa_state[\"feature_weights\"].to_dataframe().reset_index()\n",
    "    espa_weights[\"experiment\"] = row[\"experiment\"]\n",
    "    espa_weights[\"grid_level\"] = grid_level\n",
    "    espa_weights[\"model_type\"] = model_type\n",
    "    espa_weights[\"task\"] = task\n",
    "    weights.append(espa_weights)\n",
    "weights = pd.concat(weights, ignore_index=True)\n",
    "weights[\"grid_level\"] = weights[\"grid_level\"].astype(grid_level_cat)\n",
    "weights[\"feature\"] = weights[\"feature\"].apply(_parse_feature_name)\n",
    "weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3f2d039",
   "metadata": {},
   "source": [
    "## XGBoost importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "839c9105",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.03s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.07s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.07s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.23s\n",
      "Loading features from cache took 0.06s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.37s\n",
      "Loading features from cache took 0.06s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.03s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.37s\n",
      "Loading features from cache took 0.04s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.21s\n",
      "Loading features from cache took 0.06s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.03s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.03s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.03s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.24s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.03s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.21s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.03s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.03s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.03s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.23s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.21s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.03s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.23s\n",
      "Loading features from cache took 0.05s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.21s\n",
      "Loading features from cache took 0.04s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.07s\n",
      "Loading features from cache took 0.03s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.02s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.38s\n",
      "Loading features from cache took 0.05s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.03s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.01s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.02s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=binary) took 0.07s\n",
      "Loading features from cache took 0.02s\n",
      "Reading grid took 0.00s\n",
      "Getting target labels (with task=density) took 0.37s\n",
      "Loading features from cache took 0.04s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cell Area</td>\n",
       "      <td>0.006088</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Land Area</td>\n",
       "      <td>0.002149</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Water Area</td>\n",
       "      <td>0.001927</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>0.001702</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>X</td>\n",
       "      <td>0.007282</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6461</th>\n",
       "      <td>Embeddings Trend A59</td>\n",
       "      <td>0.016558</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6462</th>\n",
       "      <td>Embeddings Trend A60</td>\n",
       "      <td>0.005984</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6463</th>\n",
       "      <td>Embeddings Trend A61</td>\n",
       "      <td>0.020826</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6464</th>\n",
       "      <td>Embeddings Trend A62</td>\n",
       "      <td>0.003088</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6465</th>\n",
       "      <td>Embeddings Trend A63</td>\n",
       "      <td>0.004789</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>density</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6466 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                   feature  importance             experiment  \\\n",
       "\u001b[1;36m0\u001b[0m                Cell Area    \u001b[1;36m0.006088\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m1\u001b[0m                Land Area    \u001b[1;36m0.002149\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m2\u001b[0m               Water Area    \u001b[1;36m0.001927\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m3\u001b[0m               Land Ratio    \u001b[1;36m0.001702\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m4\u001b[0m                        X    \u001b[1;36m0.007282\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m6461\u001b[0m  Embeddings Trend A59    \u001b[1;36m0.016558\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m6462\u001b[0m  Embeddings Trend A60    \u001b[1;36m0.005984\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m6463\u001b[0m  Embeddings Trend A61    \u001b[1;36m0.020826\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m6464\u001b[0m  Embeddings Trend A62    \u001b[1;36m0.003088\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m6465\u001b[0m  Embeddings Trend A63    \u001b[1;36m0.004789\u001b[0m             AlphaEarth   \n",
       "\n",
       "            grid_level model_type     task  \n",
       "\u001b[1;36m0\u001b[0m           H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary  \n",
       "\u001b[1;36m1\u001b[0m           H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary  \n",
       "\u001b[1;36m2\u001b[0m           H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary  \n",
       "\u001b[1;36m3\u001b[0m           H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary  \n",
       "\u001b[1;36m4\u001b[0m           H3 Level \u001b[1;36m5\u001b[0m    XGBoost   binary  \n",
       "\u001b[33m...\u001b[0m                \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m6461\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density  \n",
       "\u001b[1;36m6462\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density  \n",
       "\u001b[1;36m6463\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density  \n",
       "\u001b[1;36m6464\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density  \n",
       "\u001b[1;36m6465\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m    XGBoost  density  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m6466\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_experiments = all_experiments[all_experiments[\"model_type\"] == \"XGBoost\"]\n",
    "xgboost_importances = []\n",
    "for i, row in xgboost_experiments.iterrows():\n",
    "    run: Training = row.run\n",
    "    xgb_model = run.model.named_steps[\"model\"]\n",
    "    xgboost_importances.append(\n",
    "        pd.DataFrame(\n",
    "            {\n",
    "                \"feature\": run.training_set.feature_names,\n",
    "                \"importance\": xgb_model.feature_importances_,\n",
    "                \"experiment\": row.experiment,\n",
    "                \"grid_level\": row.grid_level,\n",
    "                \"model_type\": row.model_type,\n",
    "                \"task\": row.task,\n",
    "            }\n",
    "        )\n",
    "    )\n",
    "xgboost_importances = pd.concat(xgboost_importances, ignore_index=True)\n",
    "xgboost_importances[\"grid_level\"] = xgboost_importances[\"grid_level\"].astype(grid_level_cat)\n",
    "xgboost_importances[\"feature\"] = xgboost_importances[\"feature\"].apply(_parse_feature_name)\n",
    "xgboost_importances"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28747e5a",
   "metadata": {},
   "source": [
    "## Compare importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "3f6961d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/1745987642.py:3: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  weights_corrected[\"feature_weights\"] = weights_corrected[\"feature_weights\"] * weights_corrected.groupby(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>feature_weights</th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cell Area</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Land Area</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Water Area</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>X</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3228</th>\n",
       "      <td>Embeddings Trend A59</td>\n",
       "      <td>1.030775</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3229</th>\n",
       "      <td>Embeddings Trend A60</td>\n",
       "      <td>1.030775</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>Embeddings Trend A61</td>\n",
       "      <td>1.030775</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3231</th>\n",
       "      <td>Embeddings Trend A62</td>\n",
       "      <td>1.030775</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3232</th>\n",
       "      <td>Embeddings Trend A63</td>\n",
       "      <td>1.030775</td>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3233 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                   feature  feature_weights             experiment  \\\n",
       "\u001b[1;36m0\u001b[0m                Cell Area         \u001b[1;36m1.000000\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m1\u001b[0m                Land Area         \u001b[1;36m1.000000\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m2\u001b[0m               Water Area         \u001b[1;36m1.000000\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m3\u001b[0m               Land Ratio         \u001b[1;36m1.000000\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[1;36m4\u001b[0m                        X         \u001b[1;36m1.000000\u001b[0m  ERA5-Land + ArcticDEM   \n",
       "\u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m              \u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m3228\u001b[0m  Embeddings Trend A59         \u001b[1;36m1.030775\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3229\u001b[0m  Embeddings Trend A60         \u001b[1;36m1.030775\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3230\u001b[0m  Embeddings Trend A61         \u001b[1;36m1.030775\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3231\u001b[0m  Embeddings Trend A62         \u001b[1;36m1.030775\u001b[0m             AlphaEarth   \n",
       "\u001b[1;36m3232\u001b[0m  Embeddings Trend A63         \u001b[1;36m1.030775\u001b[0m             AlphaEarth   \n",
       "\n",
       "           grid_level model_type    task  \n",
       "\u001b[1;36m0\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m1\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m2\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m4\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary  \n",
       "\u001b[33m...\u001b[0m               \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m3228\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3229\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3230\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3231\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\u001b[1;36m3232\u001b[0m  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m3233\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# weight the espa weights by the number of features, as they are normalized to sum to 1\n",
    "weights_corrected = weights.copy()\n",
    "weights_corrected[\"feature_weights\"] = weights_corrected[\"feature_weights\"] * weights_corrected.groupby(\n",
    "    [\"experiment\", \"grid_level\", \"model_type\", \"task\"]\n",
    ")[\"feature\"].transform(\"count\")\n",
    "weights_corrected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "1551ecfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>feature</th>\n",
       "      <th>importance_permutation</th>\n",
       "      <th>importance_std_permutation</th>\n",
       "      <th>importance_stability_permutation</th>\n",
       "      <th>importance_shap</th>\n",
       "      <th>importance_std_shap</th>\n",
       "      <th>importance_stability_shap</th>\n",
       "      <th>importance_xgboost</th>\n",
       "      <th>importance_espa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.050333</td>\n",
       "      <td>0.051251</td>\n",
       "      <td>1.018231</td>\n",
       "      <td>0.008998</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Land Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.068435</td>\n",
       "      <td>0.042029</td>\n",
       "      <td>0.614149</td>\n",
       "      <td>0.003141</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Water Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.006293</td>\n",
       "      <td>0.010967</td>\n",
       "      <td>1.742900</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ERA5-Land + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>X</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.036068</td>\n",
       "      <td>0.026866</td>\n",
       "      <td>0.744875</td>\n",
       "      <td>0.001100</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16160</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Trend A59</td>\n",
       "      <td>0.000571</td>\n",
       "      <td>0.000160</td>\n",
       "      <td>0.280199</td>\n",
       "      <td>0.006646</td>\n",
       "      <td>0.010132</td>\n",
       "      <td>1.524574</td>\n",
       "      <td>0.001990</td>\n",
       "      <td>1.015647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16161</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Trend A60</td>\n",
       "      <td>0.000531</td>\n",
       "      <td>0.000412</td>\n",
       "      <td>0.775533</td>\n",
       "      <td>0.007200</td>\n",
       "      <td>0.011235</td>\n",
       "      <td>1.560447</td>\n",
       "      <td>0.002005</td>\n",
       "      <td>1.015647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16162</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Trend A61</td>\n",
       "      <td>-0.000029</td>\n",
       "      <td>0.000325</td>\n",
       "      <td>11.214230</td>\n",
       "      <td>0.004422</td>\n",
       "      <td>0.005140</td>\n",
       "      <td>1.162541</td>\n",
       "      <td>0.002296</td>\n",
       "      <td>1.015647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16163</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Trend A62</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000150</td>\n",
       "      <td>13.578382</td>\n",
       "      <td>0.041373</td>\n",
       "      <td>0.034866</td>\n",
       "      <td>0.842730</td>\n",
       "      <td>0.003241</td>\n",
       "      <td>1.015647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16164</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Trend A63</td>\n",
       "      <td>0.000171</td>\n",
       "      <td>0.000515</td>\n",
       "      <td>3.013430</td>\n",
       "      <td>0.007231</td>\n",
       "      <td>0.011940</td>\n",
       "      <td>1.651231</td>\n",
       "      <td>0.002795</td>\n",
       "      <td>1.015647</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16165 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                  experiment  grid_level     model_type    task  \\\n",
       "\u001b[1;36m0\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m           eSPA  binary   \n",
       "\u001b[1;36m1\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m           eSPA  binary   \n",
       "\u001b[1;36m2\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m           eSPA  binary   \n",
       "\u001b[1;36m3\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m           eSPA  binary   \n",
       "\u001b[1;36m4\u001b[0m      ERA5-Land + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m           eSPA  binary   \n",
       "\u001b[33m...\u001b[0m                      \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m16160\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m5\u001b[0m  Random Forest  binary   \n",
       "\u001b[1;36m16161\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m5\u001b[0m  Random Forest  binary   \n",
       "\u001b[1;36m16162\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m5\u001b[0m  Random Forest  binary   \n",
       "\u001b[1;36m16163\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m5\u001b[0m  Random Forest  binary   \n",
       "\u001b[1;36m16164\u001b[0m             AlphaEarth  H3 Level \u001b[1;36m5\u001b[0m  Random Forest  binary   \n",
       "\n",
       "                    feature  importance_permutation  \\\n",
       "\u001b[1;36m0\u001b[0m                 Cell Area                \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                 Land Area                \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m                Water Area                \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                Land Ratio                \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                         X                \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[33m...\u001b[0m                     \u001b[33m...\u001b[0m                     \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m16160\u001b[0m  Embeddings Trend A59                \u001b[1;36m0.000571\u001b[0m   \n",
       "\u001b[1;36m16161\u001b[0m  Embeddings Trend A60                \u001b[1;36m0.000531\u001b[0m   \n",
       "\u001b[1;36m16162\u001b[0m  Embeddings Trend A61               \u001b[1;36m-0.000029\u001b[0m   \n",
       "\u001b[1;36m16163\u001b[0m  Embeddings Trend A62                \u001b[1;36m0.000011\u001b[0m   \n",
       "\u001b[1;36m16164\u001b[0m  Embeddings Trend A63                \u001b[1;36m0.000171\u001b[0m   \n",
       "\n",
       "       importance_std_permutation  importance_stability_permutation  \\\n",
       "\u001b[1;36m0\u001b[0m                        \u001b[1;36m0.000000\u001b[0m                               NaN   \n",
       "\u001b[1;36m1\u001b[0m                        \u001b[1;36m0.000000\u001b[0m                               NaN   \n",
       "\u001b[1;36m2\u001b[0m                        \u001b[1;36m0.000000\u001b[0m                               NaN   \n",
       "\u001b[1;36m3\u001b[0m                        \u001b[1;36m0.000000\u001b[0m                               NaN   \n",
       "\u001b[1;36m4\u001b[0m                        \u001b[1;36m0.000000\u001b[0m                               NaN   \n",
       "\u001b[33m...\u001b[0m                           \u001b[33m...\u001b[0m                               \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m16160\u001b[0m                    \u001b[1;36m0.000160\u001b[0m                          \u001b[1;36m0.280199\u001b[0m   \n",
       "\u001b[1;36m16161\u001b[0m                    \u001b[1;36m0.000412\u001b[0m                          \u001b[1;36m0.775533\u001b[0m   \n",
       "\u001b[1;36m16162\u001b[0m                    \u001b[1;36m0.000325\u001b[0m                         \u001b[1;36m11.214230\u001b[0m   \n",
       "\u001b[1;36m16163\u001b[0m                    \u001b[1;36m0.000150\u001b[0m                         \u001b[1;36m13.578382\u001b[0m   \n",
       "\u001b[1;36m16164\u001b[0m                    \u001b[1;36m0.000515\u001b[0m                          \u001b[1;36m3.013430\u001b[0m   \n",
       "\n",
       "       importance_shap  importance_std_shap  importance_stability_shap  \\\n",
       "\u001b[1;36m0\u001b[0m             \u001b[1;36m0.050333\u001b[0m             \u001b[1;36m0.051251\u001b[0m                   \u001b[1;36m1.018231\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m             \u001b[1;36m0.068435\u001b[0m             \u001b[1;36m0.042029\u001b[0m                   \u001b[1;36m0.614149\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m             \u001b[1;36m0.000000\u001b[0m             \u001b[1;36m0.000000\u001b[0m                        NaN   \n",
       "\u001b[1;36m3\u001b[0m             \u001b[1;36m0.006293\u001b[0m             \u001b[1;36m0.010967\u001b[0m                   \u001b[1;36m1.742900\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m             \u001b[1;36m0.036068\u001b[0m             \u001b[1;36m0.026866\u001b[0m                   \u001b[1;36m0.744875\u001b[0m   \n",
       "\u001b[33m...\u001b[0m                \u001b[33m...\u001b[0m                  \u001b[33m...\u001b[0m                        \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m16160\u001b[0m         \u001b[1;36m0.006646\u001b[0m             \u001b[1;36m0.010132\u001b[0m                   \u001b[1;36m1.524574\u001b[0m   \n",
       "\u001b[1;36m16161\u001b[0m         \u001b[1;36m0.007200\u001b[0m             \u001b[1;36m0.011235\u001b[0m                   \u001b[1;36m1.560447\u001b[0m   \n",
       "\u001b[1;36m16162\u001b[0m         \u001b[1;36m0.004422\u001b[0m             \u001b[1;36m0.005140\u001b[0m                   \u001b[1;36m1.162541\u001b[0m   \n",
       "\u001b[1;36m16163\u001b[0m         \u001b[1;36m0.041373\u001b[0m             \u001b[1;36m0.034866\u001b[0m                   \u001b[1;36m0.842730\u001b[0m   \n",
       "\u001b[1;36m16164\u001b[0m         \u001b[1;36m0.007231\u001b[0m             \u001b[1;36m0.011940\u001b[0m                   \u001b[1;36m1.651231\u001b[0m   \n",
       "\n",
       "       importance_xgboost  importance_espa  \n",
       "\u001b[1;36m0\u001b[0m                \u001b[1;36m0.008998\u001b[0m         \u001b[1;36m1.000000\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m                \u001b[1;36m0.003141\u001b[0m         \u001b[1;36m1.000000\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m                \u001b[1;36m0.000000\u001b[0m         \u001b[1;36m1.000000\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m                \u001b[1;36m0.002663\u001b[0m         \u001b[1;36m1.000000\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m                \u001b[1;36m0.001100\u001b[0m         \u001b[1;36m1.000000\u001b[0m  \n",
       "\u001b[33m...\u001b[0m                   \u001b[33m...\u001b[0m              \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m16160\u001b[0m            \u001b[1;36m0.001990\u001b[0m         \u001b[1;36m1.015647\u001b[0m  \n",
       "\u001b[1;36m16161\u001b[0m            \u001b[1;36m0.002005\u001b[0m         \u001b[1;36m1.015647\u001b[0m  \n",
       "\u001b[1;36m16162\u001b[0m            \u001b[1;36m0.002296\u001b[0m         \u001b[1;36m1.015647\u001b[0m  \n",
       "\u001b[1;36m16163\u001b[0m            \u001b[1;36m0.003241\u001b[0m         \u001b[1;36m1.015647\u001b[0m  \n",
       "\u001b[1;36m16164\u001b[0m            \u001b[1;36m0.002795\u001b[0m         \u001b[1;36m1.015647\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m16165\u001b[0m rows x \u001b[1;36m13\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_importances = (\n",
    "    feature_importances.merge(\n",
    "        shap_importances.drop(columns=[\"explanation_number\", \"model_type\"]),\n",
    "        on=[\"feature\", \"experiment\", \"grid_level\", \"task\"],\n",
    "        how=\"inner\",\n",
    "        suffixes=(\"_permutation\", \"_shap\"),\n",
    "    )\n",
    "    .merge(\n",
    "        xgboost_importances.rename(columns={\"importance\": \"importance_xgboost\"}).drop(columns=[\"model_type\"]),\n",
    "        on=[\"feature\", \"experiment\", \"grid_level\", \"task\"],\n",
    "        how=\"inner\",\n",
    "    )\n",
    "    .merge(\n",
    "        weights_corrected.rename(columns={\"feature_weights\": \"importance_espa\"}).drop(columns=[\"model_type\"]),\n",
    "        on=[\"feature\", \"experiment\", \"grid_level\", \"task\"],\n",
    "        how=\"inner\",\n",
    "    )\n",
    ")\n",
    "merged_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "3aa50acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/99892114.py:17: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  merged_importances.groupby([\"experiment\", \"grid_level\", \"task\"])[importance_vars]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 366744 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m2031.\u001b[0m\u001b[39m12x1600 with \u001b[0m\u001b[1;36m20\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "importance_vars = [\"importance_permutation\", \"importance_shap\", \"importance_xgboost\", \"importance_espa\"]\n",
    "g = sns.PairGrid(\n",
    "    merged_importances.sample(1000),\n",
    "    vars=importance_vars,\n",
    "    hue=\"experiment\",\n",
    "    palette=\"cubehelix\",\n",
    "    height=4,\n",
    "    diag_sharey=False,\n",
    ")\n",
    "g.map_upper(sns.scatterplot)\n",
    "g.map_lower(sns.histplot, fill=True, bins=30, hue=None)\n",
    "g.map_diag(sns.kdeplot, fill=True)\n",
    "g.add_legend()\n",
    "\n",
    "# columns: experiment, grid_level, task, level_3 and level_4\n",
    "spearman_corrs = (\n",
    "    merged_importances.groupby([\"experiment\", \"grid_level\", \"task\"])[importance_vars]\n",
    "    .corr(method=\"spearman\")\n",
    "    .stack()\n",
    "    .reset_index()\n",
    "    .rename(columns={0: \"spearman_corr\"})\n",
    ")\n",
    "for i, axs in enumerate(g.axes):\n",
    "    for j, ax in enumerate(axs):\n",
    "        ax.set_xlabel(ax.get_xlabel().replace(\"_\", \" \").title())\n",
    "        ax.set_ylabel(ax.get_ylabel().replace(\"_\", \" \").title())\n",
    "        on_diag = i == j\n",
    "        if on_diag:\n",
    "            continue\n",
    "        spearman_corrs_ij = spearman_corrs[\n",
    "            (spearman_corrs[\"level_3\"] == importance_vars[j]) & (spearman_corrs[\"level_4\"] == importance_vars[i])\n",
    "        ]\n",
    "        spearman_corr_max = spearman_corrs_ij[\"spearman_corr\"].max()\n",
    "        spearman_corr_min = spearman_corrs_ij[\"spearman_corr\"].min()\n",
    "        spearman_corr_median = spearman_corrs_ij[\"spearman_corr\"].median()\n",
    "        ax.set_title(f\"Spearman Corr: {spearman_corr_median:.2f} ({spearman_corr_min:.2f} - {spearman_corr_max:.2f})\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "499bb387",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1217096/2563627428.py:2: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  merged_importances.groupby([\"experiment\", \"grid_level\", \"task\"])[importance_vars]\n",
      "/tmp/ipykernel_1217096/2563627428.py:12: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  spearman_corrs.groupby([\"experiment\", \"grid_level\", \"task\"])\n",
      "/tmp/ipykernel_1217096/2563627428.py:13: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  .apply(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>task</th>\n",
       "      <th>level_3</th>\n",
       "      <th>level_4</th>\n",
       "      <th>spearman_corr</th>\n",
       "      <th>comparison</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_permutation</td>\n",
       "      <td>importance_espa</td>\n",
       "      <td>-0.106916</td>\n",
       "      <td>Permutation vs eSPA Weights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_shap</td>\n",
       "      <td>importance_permutation</td>\n",
       "      <td>0.115126</td>\n",
       "      <td>SHAP vs Permutation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_shap</td>\n",
       "      <td>importance_espa</td>\n",
       "      <td>-0.153365</td>\n",
       "      <td>SHAP vs eSPA Weights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_xgboost</td>\n",
       "      <td>importance_permutation</td>\n",
       "      <td>0.102382</td>\n",
       "      <td>XGBoost vs Permutation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_xgboost</td>\n",
       "      <td>importance_shap</td>\n",
       "      <td>0.883523</td>\n",
       "      <td>XGBoost vs SHAP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_shap</td>\n",
       "      <td>importance_permutation</td>\n",
       "      <td>0.100199</td>\n",
       "      <td>SHAP vs Permutation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_shap</td>\n",
       "      <td>importance_espa</td>\n",
       "      <td>0.187852</td>\n",
       "      <td>SHAP vs eSPA Weights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_xgboost</td>\n",
       "      <td>importance_permutation</td>\n",
       "      <td>0.000731</td>\n",
       "      <td>XGBoost vs Permutation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_xgboost</td>\n",
       "      <td>importance_shap</td>\n",
       "      <td>0.433511</td>\n",
       "      <td>XGBoost vs SHAP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>ERA5-Land + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>binary</td>\n",
       "      <td>importance_xgboost</td>\n",
       "      <td>importance_espa</td>\n",
       "      <td>0.496314</td>\n",
       "      <td>XGBoost vs eSPA Weights</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>156 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                       experiment  grid_level    task  \\\n",
       "\u001b[1;36m0\u001b[0m                                      AlphaEarth  H3 Level \u001b[1;36m3\u001b[0m  binary   \n",
       "\u001b[1;36m1\u001b[0m                                      AlphaEarth  H3 Level \u001b[1;36m3\u001b[0m  binary   \n",
       "\u001b[1;36m2\u001b[0m                                      AlphaEarth  H3 Level \u001b[1;36m3\u001b[0m  binary   \n",
       "\u001b[1;36m3\u001b[0m                                      AlphaEarth  H3 Level \u001b[1;36m3\u001b[0m  binary   \n",
       "\u001b[1;36m4\u001b[0m                                      AlphaEarth  H3 Level \u001b[1;36m3\u001b[0m  binary   \n",
       "..                                            \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m151\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  binary   \n",
       "\u001b[1;36m152\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  binary   \n",
       "\u001b[1;36m153\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  binary   \n",
       "\u001b[1;36m154\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  binary   \n",
       "\u001b[1;36m155\u001b[0m  ERA5-Land + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m  binary   \n",
       "\n",
       "                    level_3                 level_4  spearman_corr  \\\n",
       "\u001b[1;36m0\u001b[0m    importance_permutation         importance_espa      \u001b[1;36m-0.106916\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m           importance_shap  importance_permutation       \u001b[1;36m0.115126\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m           importance_shap         importance_espa      \u001b[1;36m-0.153365\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m        importance_xgboost  importance_permutation       \u001b[1;36m0.102382\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m        importance_xgboost         importance_shap       \u001b[1;36m0.883523\u001b[0m   \n",
       "..                      \u001b[33m...\u001b[0m                     \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m151\u001b[0m         importance_shap  importance_permutation       \u001b[1;36m0.100199\u001b[0m   \n",
       "\u001b[1;36m152\u001b[0m         importance_shap         importance_espa       \u001b[1;36m0.187852\u001b[0m   \n",
       "\u001b[1;36m153\u001b[0m      importance_xgboost  importance_permutation       \u001b[1;36m0.000731\u001b[0m   \n",
       "\u001b[1;36m154\u001b[0m      importance_xgboost         importance_shap       \u001b[1;36m0.433511\u001b[0m   \n",
       "\u001b[1;36m155\u001b[0m      importance_xgboost         importance_espa       \u001b[1;36m0.496314\u001b[0m   \n",
       "\n",
       "                      comparison  \n",
       "\u001b[1;36m0\u001b[0m    Permutation vs eSPA Weights  \n",
       "\u001b[1;36m1\u001b[0m            SHAP vs Permutation  \n",
       "\u001b[1;36m2\u001b[0m           SHAP vs eSPA Weights  \n",
       "\u001b[1;36m3\u001b[0m         XGBoost vs Permutation  \n",
       "\u001b[1;36m4\u001b[0m                XGBoost vs SHAP  \n",
       "..                           \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m151\u001b[0m          SHAP vs Permutation  \n",
       "\u001b[1;36m152\u001b[0m         SHAP vs eSPA Weights  \n",
       "\u001b[1;36m153\u001b[0m       XGBoost vs Permutation  \n",
       "\u001b[1;36m154\u001b[0m              XGBoost vs SHAP  \n",
       "\u001b[1;36m155\u001b[0m      XGBoost vs eSPA Weights  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m156\u001b[0m rows x \u001b[1;36m7\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearman_corrs = (\n",
    "    merged_importances.groupby([\"experiment\", \"grid_level\", \"task\"])[importance_vars]\n",
    "    .corr(method=\"spearman\")\n",
    "    .stack()\n",
    "    .reset_index()\n",
    "    .rename(columns={0: \"spearman_corr\"})\n",
    ")\n",
    "# Remove all comparisons that are the same (e.g. permutation vs permutation)\n",
    "spearman_corrs = spearman_corrs[~(spearman_corrs[\"level_3\"] == spearman_corrs[\"level_4\"])]\n",
    "# Remove all comparisons that are duplicated, but reverse (e.g. permutation vs shap and shap vs permutation)\n",
    "spearman_corrs = (\n",
    "    spearman_corrs.groupby([\"experiment\", \"grid_level\", \"task\"])\n",
    "    .apply(\n",
    "        lambda df: df[\n",
    "            ~df.apply(lambda row: (row[\"level_3\"], row[\"level_4\"]) < (row[\"level_4\"], row[\"level_3\"]), axis=1)\n",
    "        ]\n",
    "    )\n",
    "    .reset_index(drop=True)\n",
    ")\n",
    "# Combine the level_3 and level_4 columns into a single column called \"comparison\" with values like \"permutation vs shap\", \"permutation vs xgboost\", etc.\n",
    "spearman_corrs[\"comparison\"] = (\n",
    "    spearman_corrs[\"level_3\"].str.replace(\"importance_\", \"\").str.title()\n",
    "    + \" vs \"\n",
    "    + spearman_corrs[\"level_4\"].str.replace(\"importance_\", \"\").str.title()\n",
    ")\n",
    "\n",
    "\n",
    "def _parse_comparison_name(comparison_name: str) -> str:\n",
    "    comparison_name = (\n",
    "        comparison_name.replace(\"Espa\", \"eSPA Weights\")\n",
    "        .replace(\"Xgboost\", \"XGBoost\")\n",
    "        .replace(\"Shap\", \"SHAP\")\n",
    "        .replace(\"Permutation\", \"Permutation\")\n",
    "    )\n",
    "    return comparison_name\n",
    "\n",
    "\n",
    "spearman_corrs[\"comparison\"] = spearman_corrs[\"comparison\"].apply(_parse_comparison_name)\n",
    "spearman_corrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "03f609d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 281700 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 120\u001b[0m\u001b[1;36m0x800\u001b[0m\u001b[39m with \u001b[0m\u001b[1;36m6\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import seaborn.objects as so\n",
    "\n",
    "p = (\n",
    "    so.Plot(spearman_corrs, x=\"grid_level\", y=\"spearman_corr\", color=\"experiment\")\n",
    "    .facet(\"comparison\", wrap=3)\n",
    "    .add(so.Dot())\n",
    "    .add(so.Line())\n",
    "    .scale(color=\"cubehelix\")\n",
    "    .layout(size=(12, 8), engine=\"constrained\")\n",
    "    .label(x=\"Grid\", y=\"Spearman Correlation\", legend=\"Experiment\")\n",
    "    .theme(plt.rcParams)\n",
    ")\n",
    "fig = p.plot()._figure\n",
    "for ax in fig.axes:\n",
    "    for label in ax.get_xticklabels():\n",
    "        # Align x-axis labels right and rotate them by 45 degrees\n",
    "        label.set_horizontalalignment(\"right\")\n",
    "        label.set_rotation(45)\n",
    "\n",
    "    # Draw a box signaling the grid-resolutions\n",
    "    ax.axvspan(-0.5, 3.5, color=\"#2e4a9e\", alpha=0.08)\n",
    "    ax.axvspan(3.5, 6.5, color=\"#005a00\", alpha=0.08)\n",
    "    ax.axvspan(6.5, 8.5, color=\"#6b4700\", alpha=0.08)\n",
    "\n",
    "    # Add text labels for the grid resolutions\n",
    "    ax.text(1.5, -0.75, \"Low\", color=\"#2e4a9e\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "    ax.text(5, -0.75, \"Medium\", color=\"#005a00\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "    ax.text(7.5, -0.75, \"High\", color=\"#6b4700\", fontsize=12, ha=\"center\", va=\"center\")\n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "895f8d80",
   "metadata": {},
   "source": [
    "## eSPA weights vs. Feature Importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "40565e03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>importance_std</th>\n",
       "      <th>feature_weights</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Land Area</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Water Area</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Land Ratio</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>X</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3228</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings Trend A59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3229</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings Trend A60</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings Trend A61</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3231</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings Trend A62</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3232</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings Trend A63</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007692</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3233 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "            experiment       grid_level model_type    task  \\\n",
       "\u001b[1;36m0\u001b[0m     ERA5 + ArcticDEM       H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m1\u001b[0m     ERA5 + ArcticDEM       H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m2\u001b[0m     ERA5 + ArcticDEM       H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m3\u001b[0m     ERA5 + ArcticDEM       H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m4\u001b[0m     ERA5 + ArcticDEM       H3 Level \u001b[1;36m3\u001b[0m       eSPA  binary   \n",
       "\u001b[33m...\u001b[0m                \u001b[33m...\u001b[0m              \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m3228\u001b[0m        AlphaEarth  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m3229\u001b[0m        AlphaEarth  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m3230\u001b[0m        AlphaEarth  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m3231\u001b[0m        AlphaEarth  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary   \n",
       "\u001b[1;36m3232\u001b[0m        AlphaEarth  HEALPix Level \u001b[1;36m8\u001b[0m       eSPA  binary   \n",
       "\n",
       "                              feature  importance  importance_std  \\\n",
       "\u001b[1;36m0\u001b[0m                           Cell Area         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                           Land Area         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m                          Water Area         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                          Land Ratio         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                                   X         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[33m...\u001b[0m                               \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m3228\u001b[0m  Embeddings Embeddings Trend A59         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m3229\u001b[0m  Embeddings Embeddings Trend A60         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m3230\u001b[0m  Embeddings Embeddings Trend A61         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m3231\u001b[0m  Embeddings Embeddings Trend A62         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\u001b[1;36m3232\u001b[0m  Embeddings Embeddings Trend A63         \u001b[1;36m0.0\u001b[0m             \u001b[1;36m0.0\u001b[0m   \n",
       "\n",
       "      feature_weights  \n",
       "\u001b[1;36m0\u001b[0m            \u001b[1;36m0.005587\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m            \u001b[1;36m0.005587\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m            \u001b[1;36m0.005587\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m            \u001b[1;36m0.005587\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m            \u001b[1;36m0.005587\u001b[0m  \n",
       "\u001b[33m...\u001b[0m               \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m3228\u001b[0m         \u001b[1;36m0.007692\u001b[0m  \n",
       "\u001b[1;36m3229\u001b[0m         \u001b[1;36m0.007692\u001b[0m  \n",
       "\u001b[1;36m3230\u001b[0m         \u001b[1;36m0.007692\u001b[0m  \n",
       "\u001b[1;36m3231\u001b[0m         \u001b[1;36m0.007692\u001b[0m  \n",
       "\u001b[1;36m3232\u001b[0m         \u001b[1;36m0.007692\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m3233\u001b[0m rows x \u001b[1;36m8\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "espa_importances_weights = feature_importances_binary[feature_importances_binary[\"model_type\"] == \"eSPA\"].merge(\n",
    "    weights[weights[\"task\"] == \"binary\"],\n",
    "    on=[\"experiment\", \"grid_level\", \"model_type\", \"task\", \"feature\"],\n",
    "    how=\"inner\",\n",
    ")\n",
    "espa_importances_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "ab18fa26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 222656 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m2163.\u001b[0m\u001b[39m12x1000 with \u001b[0m\u001b[1;36m4\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "espa_importances_weights_h3_default = espa_importances_weights[\n",
    "    espa_importances_weights[\"grid_level\"].str.startswith(\"H3\")\n",
    "    & (espa_importances_weights[\"experiment\"] == \"ERA5 + ArcticDEM\")\n",
    "].copy()\n",
    "espa_importances_weights_h3_default[\"grid_level\"] = espa_importances_weights_h3_default[\n",
    "    \"grid_level\"\n",
    "].cat.remove_unused_categories()\n",
    "g = sns.relplot(\n",
    "    espa_importances_weights_h3_default,\n",
    "    x=\"feature_weights\",\n",
    "    y=\"importance\",\n",
    "    # kind=\"kde\",\n",
    "    hue=\"grid_level\",\n",
    "    # fill=True,\n",
    "    palette=\"cubehelix\",\n",
    "    col=\"grid_level\",\n",
    "    col_wrap=2,\n",
    "    height=5,\n",
    "    aspect=2,\n",
    "    facet_kws={\"sharex\": False, \"sharey\": False},\n",
    ")\n",
    "\n",
    "g._legend.remove()\n",
    "# Titelize the axes titles and axis titles\n",
    "for ax in g.axes.flat:\n",
    "    ax.set_xlabel(ax.get_xlabel().replace(\"feature_weights\", \"Feature Weights\").title())\n",
    "    ax.set_ylabel(ax.get_ylabel().replace(\"importance\", \"Feature Importance\").title())\n",
    "\n",
    "    # Calculate spearman correlation\n",
    "    grid_level = ax.get_title().replace(\"grid_level = \", \"\")\n",
    "    corr = (\n",
    "        espa_importances_weights_h3_default[espa_importances_weights_h3_default[\"grid_level\"] == grid_level][\n",
    "            [\"feature_weights\", \"importance\"]\n",
    "        ]\n",
    "        .corr(method=\"spearman\")\n",
    "        .iloc[1, 0]\n",
    "    )\n",
    "    ax.set_title(f\"{grid_level.title()} (Spearman Corr: {corr:.2f})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "82ef9823",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_319674/2473424046.py:3: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  espa_importances_weights.groupby([\"experiment\", \"grid_level\"])\n",
      "/tmp/ipykernel_319674/2473424046.py:4: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  .apply(lambda df: df[\"feature_weights\"].corr(df[\"importance\"], method=\"spearman\"))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>correlation</th>\n",
       "      <th>correlation_abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>0.000755</td>\n",
       "      <td>0.000755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>0.171024</td>\n",
       "      <td>0.171024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>0.542608</td>\n",
       "      <td>0.542608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>-0.561762</td>\n",
       "      <td>0.561762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>-0.386149</td>\n",
       "      <td>0.386149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>0.108726</td>\n",
       "      <td>0.108726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>-0.004783</td>\n",
       "      <td>0.004783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>0.067034</td>\n",
       "      <td>0.067034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>0.274208</td>\n",
       "      <td>0.274208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>0.219635</td>\n",
       "      <td>0.219635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>0.045768</td>\n",
       "      <td>0.045768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>-0.126624</td>\n",
       "      <td>0.126624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>-0.153669</td>\n",
       "      <td>0.153669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>0.113431</td>\n",
       "      <td>0.113431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>-0.363553</td>\n",
       "      <td>0.363553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>-0.171249</td>\n",
       "      <td>0.171249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>0.238123</td>\n",
       "      <td>0.238123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>-0.389700</td>\n",
       "      <td>0.389700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>-0.175349</td>\n",
       "      <td>0.175349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>-0.161969</td>\n",
       "      <td>0.161969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>-0.014071</td>\n",
       "      <td>0.014071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ERA5 + ArcticDEM (Cluster Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>-0.310463</td>\n",
       "      <td>0.310463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>-0.305828</td>\n",
       "      <td>0.305828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>-0.282589</td>\n",
       "      <td>0.282589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>-0.116330</td>\n",
       "      <td>0.116330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>-0.394818</td>\n",
       "      <td>0.394818</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                 experiment        grid_level  correlation  \\\n",
       "\u001b[1;36m0\u001b[0m                                AlphaEarth        H3 Level \u001b[1;36m3\u001b[0m     \u001b[1;36m0.000755\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                                AlphaEarth   HEALPix Level \u001b[1;36m6\u001b[0m     \u001b[1;36m0.171024\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m                                AlphaEarth   HEALPix Level \u001b[1;36m7\u001b[0m     \u001b[1;36m0.542608\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m                                AlphaEarth        H3 Level \u001b[1;36m4\u001b[0m    \u001b[1;36m-0.561762\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                                AlphaEarth   HEALPix Level \u001b[1;36m8\u001b[0m    \u001b[1;36m-0.386149\u001b[0m   \n",
       "\u001b[1;36m5\u001b[0m                                AlphaEarth        H3 Level \u001b[1;36m5\u001b[0m     \u001b[1;36m0.108726\u001b[0m   \n",
       "\u001b[1;36m6\u001b[0m                                AlphaEarth   HEALPix Level \u001b[1;36m9\u001b[0m    \u001b[1;36m-0.004783\u001b[0m   \n",
       "\u001b[1;36m7\u001b[0m                                AlphaEarth  HEALPix Level \u001b[1;36m10\u001b[0m     \u001b[1;36m0.067034\u001b[0m   \n",
       "\u001b[1;36m8\u001b[0m                                AlphaEarth        H3 Level \u001b[1;36m6\u001b[0m     \u001b[1;36m0.274208\u001b[0m   \n",
       "\u001b[1;36m9\u001b[0m                          ERA5 + ArcticDEM        H3 Level \u001b[1;36m3\u001b[0m     \u001b[1;36m0.219635\u001b[0m   \n",
       "\u001b[1;36m10\u001b[0m                         ERA5 + ArcticDEM   HEALPix Level \u001b[1;36m6\u001b[0m     \u001b[1;36m0.045768\u001b[0m   \n",
       "\u001b[1;36m11\u001b[0m                         ERA5 + ArcticDEM   HEALPix Level \u001b[1;36m7\u001b[0m    \u001b[1;36m-0.126624\u001b[0m   \n",
       "\u001b[1;36m12\u001b[0m                         ERA5 + ArcticDEM        H3 Level \u001b[1;36m4\u001b[0m    \u001b[1;36m-0.153669\u001b[0m   \n",
       "\u001b[1;36m13\u001b[0m                         ERA5 + ArcticDEM   HEALPix Level \u001b[1;36m8\u001b[0m     \u001b[1;36m0.113431\u001b[0m   \n",
       "\u001b[1;36m14\u001b[0m                         ERA5 + ArcticDEM        H3 Level \u001b[1;36m5\u001b[0m    \u001b[1;36m-0.363553\u001b[0m   \n",
       "\u001b[1;36m15\u001b[0m                         ERA5 + ArcticDEM   HEALPix Level \u001b[1;36m9\u001b[0m    \u001b[1;36m-0.171249\u001b[0m   \n",
       "\u001b[1;36m16\u001b[0m                         ERA5 + ArcticDEM  HEALPix Level \u001b[1;36m10\u001b[0m     \u001b[1;36m0.238123\u001b[0m   \n",
       "\u001b[1;36m17\u001b[0m                         ERA5 + ArcticDEM        H3 Level \u001b[1;36m6\u001b[0m    \u001b[1;36m-0.389700\u001b[0m   \n",
       "\u001b[1;36m18\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m3\u001b[0m    \u001b[1;36m-0.175349\u001b[0m   \n",
       "\u001b[1;36m19\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m4\u001b[0m    \u001b[1;36m-0.161969\u001b[0m   \n",
       "\u001b[1;36m20\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m    \u001b[1;36m-0.014071\u001b[0m   \n",
       "\u001b[1;36m21\u001b[0m     ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m    \u001b[1;36m-0.310463\u001b[0m   \n",
       "\u001b[1;36m22\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m3\u001b[0m    \u001b[1;36m-0.305828\u001b[0m   \n",
       "\u001b[1;36m23\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m4\u001b[0m    \u001b[1;36m-0.282589\u001b[0m   \n",
       "\u001b[1;36m24\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m    \u001b[1;36m-0.116330\u001b[0m   \n",
       "\u001b[1;36m25\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m    \u001b[1;36m-0.394818\u001b[0m   \n",
       "\n",
       "    correlation_abs  \n",
       "\u001b[1;36m0\u001b[0m          \u001b[1;36m0.000755\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m          \u001b[1;36m0.171024\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m          \u001b[1;36m0.542608\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m          \u001b[1;36m0.561762\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m          \u001b[1;36m0.386149\u001b[0m  \n",
       "\u001b[1;36m5\u001b[0m          \u001b[1;36m0.108726\u001b[0m  \n",
       "\u001b[1;36m6\u001b[0m          \u001b[1;36m0.004783\u001b[0m  \n",
       "\u001b[1;36m7\u001b[0m          \u001b[1;36m0.067034\u001b[0m  \n",
       "\u001b[1;36m8\u001b[0m          \u001b[1;36m0.274208\u001b[0m  \n",
       "\u001b[1;36m9\u001b[0m          \u001b[1;36m0.219635\u001b[0m  \n",
       "\u001b[1;36m10\u001b[0m         \u001b[1;36m0.045768\u001b[0m  \n",
       "\u001b[1;36m11\u001b[0m         \u001b[1;36m0.126624\u001b[0m  \n",
       "\u001b[1;36m12\u001b[0m         \u001b[1;36m0.153669\u001b[0m  \n",
       "\u001b[1;36m13\u001b[0m         \u001b[1;36m0.113431\u001b[0m  \n",
       "\u001b[1;36m14\u001b[0m         \u001b[1;36m0.363553\u001b[0m  \n",
       "\u001b[1;36m15\u001b[0m         \u001b[1;36m0.171249\u001b[0m  \n",
       "\u001b[1;36m16\u001b[0m         \u001b[1;36m0.238123\u001b[0m  \n",
       "\u001b[1;36m17\u001b[0m         \u001b[1;36m0.389700\u001b[0m  \n",
       "\u001b[1;36m18\u001b[0m         \u001b[1;36m0.175349\u001b[0m  \n",
       "\u001b[1;36m19\u001b[0m         \u001b[1;36m0.161969\u001b[0m  \n",
       "\u001b[1;36m20\u001b[0m         \u001b[1;36m0.014071\u001b[0m  \n",
       "\u001b[1;36m21\u001b[0m         \u001b[1;36m0.310463\u001b[0m  \n",
       "\u001b[1;36m22\u001b[0m         \u001b[1;36m0.305828\u001b[0m  \n",
       "\u001b[1;36m23\u001b[0m         \u001b[1;36m0.282589\u001b[0m  \n",
       "\u001b[1;36m24\u001b[0m         \u001b[1;36m0.116330\u001b[0m  \n",
       "\u001b[1;36m25\u001b[0m         \u001b[1;36m0.394818\u001b[0m  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation between feature importance and feature weights for each grid level and experiment\n",
    "correlations = (\n",
    "    espa_importances_weights.groupby([\"experiment\", \"grid_level\"])\n",
    "    .apply(lambda df: df[\"feature_weights\"].corr(df[\"importance\"], method=\"spearman\"))\n",
    "    .reset_index(name=\"correlation\")\n",
    ")\n",
    "correlations[\"correlation_abs\"] = correlations[\"correlation\"].abs()\n",
    "correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4d00c224",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/raid/scratch/tohoel001/repositories/entropice/.pixi/envs/default/lib/python3.13/site-packages/seaborn/axisgrid.py:123: UserWarning: Tight layout not applied. The left and right margins cannot be made large enough to accommodate all Axes decorations.\n",
      "  self._figure.tight_layout(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 50204 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m480.\u001b[0m\u001b[39m125x400 with \u001b[0m\u001b[1;36m1\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.relplot(\n",
    "    correlations[correlations[\"experiment\"] == \"ERA5 + ArcticDEM\"],\n",
    "    x=\"experiment\",\n",
    "    y=\"grid_level\",\n",
    "    size=\"correlation_abs\",\n",
    "    hue=\"correlation\",\n",
    "    hue_norm=(-0.3, 0.3),\n",
    "    palette=\"vlag\",\n",
    "    edgecolor=\".7\",\n",
    "    sizes=(100, 250),\n",
    "    size_norm=(0, 0.3),\n",
    "    legend=\"auto\",\n",
    "    height=4,\n",
    "    aspect=0.4,\n",
    ")\n",
    "# g.set(xlabel=\"\", ylabel=\"\", aspect=\"equal\")\n",
    "g.despine(left=True, bottom=True)\n",
    "# Disable the x-axis labels\n",
    "g.set(xticklabels=[])\n",
    "# Disable the axis titles\n",
    "g.set(xlabel=\"\", ylabel=\"\")\n",
    "# Redo the legend\n",
    "legend_ticks = [-0.3, -0.15, 0, 0.15, 0.3]\n",
    "legend_labels = [f\"{tick:.2f}\" for tick in legend_ticks]\n",
    "legend_colors = sns.color_palette(\"vlag\", n_colors=len(legend_ticks))\n",
    "legend_sizes = [250, 175, 100, 175, 250]\n",
    "legend_handles = {\n",
    "    lbl: plt.Line2D([0], [0], marker=\"o\", color=\"w\", markerfacecolor=color, markersize=np.sqrt(size))\n",
    "    for lbl, color, size in zip(legend_labels, legend_colors, legend_sizes)\n",
    "}\n",
    "g._legend.remove()\n",
    "g.add_legend(legend_handles, title=\"\")\n",
    "# Disable the grid\n",
    "# g.ax.grid(False)\n",
    "# Write the correlation values on the plot\n",
    "for i, row in correlations[correlations[\"experiment\"] == \"ERA5 + ArcticDEM\"].iterrows():\n",
    "    x = 0.02\n",
    "    y = row[\"grid_level\"]\n",
    "    g.ax.text(\n",
    "        x,\n",
    "        y,\n",
    "        f\"{row['correlation']:.2f}\",\n",
    "        ha=\"center\",\n",
    "        va=\"bottom\",\n",
    "        color=\"black\",\n",
    "        fontsize=10,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd46ba3",
   "metadata": {},
   "source": [
    "## Feature Importance Stability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0fd0e804",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_319674/2186224534.py:17: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  .groupby([\"experiment\", \"feature\", \"grid_level\", \"task\"])\n",
      "/tmp/ipykernel_319674/2186224534.py:10: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  stability = np.std(samples) / np.abs(np.mean(samples))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>feature</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>task</th>\n",
       "      <th>importance_stability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>binary</td>\n",
       "      <td>3.829527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>density</td>\n",
       "      <td>2.100372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>binary</td>\n",
       "      <td>1.708653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>density</td>\n",
       "      <td>1.502922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>binary</td>\n",
       "      <td>1.382879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6461</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>Y</td>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>density</td>\n",
       "      <td>2.694460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6462</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>Y</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>binary</td>\n",
       "      <td>0.842387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6463</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>Y</td>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>density</td>\n",
       "      <td>0.800375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6464</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>Y</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>binary</td>\n",
       "      <td>0.531528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6465</th>\n",
       "      <td>ERA5 + ArcticDEM (Univariate Selection)</td>\n",
       "      <td>Y</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>density</td>\n",
       "      <td>0.674516</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3336 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                   experiment    feature  grid_level     task  \\\n",
       "\u001b[1;36m0\u001b[0m                                  AlphaEarth  Cell Area  H3 Level \u001b[1;36m3\u001b[0m   binary   \n",
       "\u001b[1;36m1\u001b[0m                                  AlphaEarth  Cell Area  H3 Level \u001b[1;36m3\u001b[0m  density   \n",
       "\u001b[1;36m6\u001b[0m                                  AlphaEarth  Cell Area  H3 Level \u001b[1;36m4\u001b[0m   binary   \n",
       "\u001b[1;36m7\u001b[0m                                  AlphaEarth  Cell Area  H3 Level \u001b[1;36m4\u001b[0m  density   \n",
       "\u001b[1;36m10\u001b[0m                                 AlphaEarth  Cell Area  H3 Level \u001b[1;36m5\u001b[0m   binary   \n",
       "\u001b[33m...\u001b[0m                                       \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m6461\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m          Y  H3 Level \u001b[1;36m4\u001b[0m  density   \n",
       "\u001b[1;36m6462\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m          Y  H3 Level \u001b[1;36m5\u001b[0m   binary   \n",
       "\u001b[1;36m6463\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m          Y  H3 Level \u001b[1;36m5\u001b[0m  density   \n",
       "\u001b[1;36m6464\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m          Y  H3 Level \u001b[1;36m6\u001b[0m   binary   \n",
       "\u001b[1;36m6465\u001b[0m  ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m          Y  H3 Level \u001b[1;36m6\u001b[0m  density   \n",
       "\n",
       "      importance_stability  \n",
       "\u001b[1;36m0\u001b[0m                 \u001b[1;36m3.829527\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m                 \u001b[1;36m2.100372\u001b[0m  \n",
       "\u001b[1;36m6\u001b[0m                 \u001b[1;36m1.708653\u001b[0m  \n",
       "\u001b[1;36m7\u001b[0m                 \u001b[1;36m1.502922\u001b[0m  \n",
       "\u001b[1;36m10\u001b[0m                \u001b[1;36m1.382879\u001b[0m  \n",
       "\u001b[33m...\u001b[0m                    \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m6461\u001b[0m              \u001b[1;36m2.694460\u001b[0m  \n",
       "\u001b[1;36m6462\u001b[0m              \u001b[1;36m0.842387\u001b[0m  \n",
       "\u001b[1;36m6463\u001b[0m              \u001b[1;36m0.800375\u001b[0m  \n",
       "\u001b[1;36m6464\u001b[0m              \u001b[1;36m0.531528\u001b[0m  \n",
       "\u001b[1;36m6465\u001b[0m              \u001b[1;36m0.674516\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m3336\u001b[0m rows x \u001b[1;36m5\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def _get_importance_stability(feature_importances_group: pd.DataFrame) -> float:\n",
    "    samples = []\n",
    "    for i, row in feature_importances_group.iterrows():\n",
    "        # Create a sample distribution of the \"importance\" and \"importance_std\" by sampling from a normal distribution\n",
    "        gen = np.random.default_rng(seed=42)\n",
    "        importance_samples = gen.normal(loc=row[\"importance\"], scale=row[\"importance_std\"], size=10000)\n",
    "        samples.append(importance_samples)\n",
    "    # Calculate stability\n",
    "    samples = np.concatenate(samples)\n",
    "    stability = np.std(samples) / np.abs(np.mean(samples))\n",
    "    return stability\n",
    "\n",
    "\n",
    "feature_importances_stability = (\n",
    "    # feature_importances\n",
    "    feature_importances[feature_importances[\"model_type\"].isin([\"XGBoost\", \"Random Forest\", \"kNN\"])]\n",
    "    .groupby([\"experiment\", \"feature\", \"grid_level\", \"task\"])\n",
    "    .apply(_get_importance_stability, include_groups=False)  # ty:ignore[no-matching-overload]\n",
    "    .reset_index(name=\"importance_stability\")\n",
    ")\n",
    "feature_importances_stability = feature_importances_stability[\n",
    "    feature_importances_stability[\"grid_level\"].str.startswith(\"H3\")\n",
    "]\n",
    "feature_importances_stability[\"grid_level\"] = feature_importances_stability[\"grid_level\"].cat.remove_unused_categories()\n",
    "feature_importances_stability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e5e7264c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Experiment: AlphaEarth, Shapiro-Wilk stat: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0786</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0000</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Experiment: AlphaEarth, Shapiro-Wilk stat: \u001b[1;36m0.0786\u001b[0m, p-value: \u001b[1;36m0.0000\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Experiment: ERA5 + ArcticDEM, Shapiro-Wilk stat: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1795</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0000</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Experiment: ERA5 + ArcticDEM, Shapiro-Wilk stat: \u001b[1;36m0.1795\u001b[0m, p-value: \u001b[1;36m0.0000\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Cluster Selection<span style=\"font-weight: bold\">)</span>, Shapiro-Wilk stat: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2352</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0000</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m, Shapiro-Wilk stat: \u001b[1;36m0.2352\u001b[0m, p-value: \u001b[1;36m0.0000\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Univariate Selection<span style=\"font-weight: bold\">)</span>, Shapiro-Wilk stat: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0923</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0000</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m, Shapiro-Wilk stat: \u001b[1;36m0.0923\u001b[0m, p-value: \u001b[1;36m0.0000\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compute shapiro wilk test for normality of the importance stability values\n",
    "for experiment in feature_importances_stability[\"experiment\"].unique():\n",
    "    experiment_stability = feature_importances_stability[feature_importances_stability[\"experiment\"] == experiment][\n",
    "        \"importance_stability\"\n",
    "    ]\n",
    "    stat, p_value = shapiro(experiment_stability.dropna())\n",
    "    print(f\"Experiment: {experiment}, Shapiro-Wilk stat: {stat:.4f}, p-value: {p_value:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "407e2c27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 156556 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m2240.\u001b[0m\u001b[39m75x600 with \u001b[0m\u001b[1;36m3\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_importances_stability_filtered = feature_importances_stability[\n",
    "    feature_importances_stability[\"grid_level\"].isin([\"H3 Level 4\", \"H3 Level 5\", \"H3 Level 3\"])\n",
    "    & (feature_importances_stability[\"task\"] == \"binary\")\n",
    "].copy()\n",
    "feature_importances_stability_filtered[\"grid_level\"] = feature_importances_stability_filtered[\n",
    "    \"grid_level\"\n",
    "].cat.remove_unused_categories()\n",
    "g = sns.displot(\n",
    "    feature_importances_stability_filtered,\n",
    "    x=\"importance_stability\",\n",
    "    hue=\"experiment\",\n",
    "    # kind=\"kde\",\n",
    "    fill=True,\n",
    "    kde=True,\n",
    "    palette=\"viridis\",\n",
    "    col=\"grid_level\",\n",
    "    height=6,\n",
    "    log_scale=True,\n",
    ")\n",
    "\n",
    "# Rename the axis titles\n",
    "for ax in g.axes.flat:\n",
    "    ax.set_xlabel(ax.get_xlabel().replace(\"importance_stability\", r\"Importance Stability $\\frac{\\sigma}{\\mu}$\"))\n",
    "    ax.set_title(ax.get_title().replace(\"grid_level = \", \"\").title())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "088b3b1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Grid Level: H3 Level <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Grid Level: H3 Level \u001b[1;36m3\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: AlphaEarth, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.9745</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1613</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: AlphaEarth, ANOVA F-statistic: \u001b[1;36m1.9745\u001b[0m, p-value: \u001b[1;36m0.1613\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Cluster Selection<span style=\"font-weight: bold\">)</span>, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7780</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3788</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m, ANOVA F-statistic: \u001b[1;36m0.7780\u001b[0m, p-value: \u001b[1;36m0.3788\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Univariate Selection<span style=\"font-weight: bold\">)</span>, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2241</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6365</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m, ANOVA F-statistic: \u001b[1;36m0.2241\u001b[0m, p-value: \u001b[1;36m0.6365\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Grid Level: H3 Level <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Grid Level: H3 Level \u001b[1;36m4\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: AlphaEarth, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.1320</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2884</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: AlphaEarth, ANOVA F-statistic: \u001b[1;36m1.1320\u001b[0m, p-value: \u001b[1;36m0.2884\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Cluster Selection<span style=\"font-weight: bold\">)</span>, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.6378</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1062</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m, ANOVA F-statistic: \u001b[1;36m2.6378\u001b[0m, p-value: \u001b[1;36m0.1062\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Univariate Selection<span style=\"font-weight: bold\">)</span>, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7497</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3878</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m, ANOVA F-statistic: \u001b[1;36m0.7497\u001b[0m, p-value: \u001b[1;36m0.3878\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Grid Level: H3 Level <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Grid Level: H3 Level \u001b[1;36m5\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: AlphaEarth, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13.0858</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0003</span> <span style=\"font-weight: bold\">[</span>Significant<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: AlphaEarth, ANOVA F-statistic: \u001b[1;36m13.0858\u001b[0m, p-value: \u001b[1;36m0.0003\u001b[0m \u001b[1m[\u001b[0mSignificant\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Cluster Selection<span style=\"font-weight: bold\">)</span>, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.5899</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0594</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m, ANOVA F-statistic: \u001b[1;36m3.5899\u001b[0m, p-value: \u001b[1;36m0.0594\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">  Experiment: ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Univariate Selection<span style=\"font-weight: bold\">)</span>, ANOVA F-statistic: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5.5879</span>, p-value: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0189</span> <span style=\"font-weight: bold\">[</span>Significant<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "  Experiment: ERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m, ANOVA F-statistic: \u001b[1;36m5.5879\u001b[0m, p-value: \u001b[1;36m0.0189\u001b[0m \u001b[1m[\u001b[0mSignificant\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "Typst Table:\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "Typst Table:\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">table.cell</span><span style=\"font-weight: bold\">(</span>strong<span style=\"font-weight: bold\">[</span>H3 Level <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">]</span>, colspan: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">)</span>,\n",
       "<span style=\"font-weight: bold\">[</span>AlphaEarth<span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.9745</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1613</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"font-weight: bold\">[</span>ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Cluster Selection<span style=\"font-weight: bold\">)]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7780</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3788</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"font-weight: bold\">[</span>ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Univariate Selection<span style=\"font-weight: bold\">)]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2241</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6365</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">table.cell</span><span style=\"font-weight: bold\">(</span>strong<span style=\"font-weight: bold\">[</span>H3 Level <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"font-weight: bold\">]</span>, colspan: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">)</span>,\n",
       "<span style=\"font-weight: bold\">[</span>AlphaEarth<span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.1320</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2884</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"font-weight: bold\">[</span>ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Cluster Selection<span style=\"font-weight: bold\">)]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.6378</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1062</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"font-weight: bold\">[</span>ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Univariate Selection<span style=\"font-weight: bold\">)]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7497</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3878</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">table.cell</span><span style=\"font-weight: bold\">(</span>strong<span style=\"font-weight: bold\">[</span>H3 Level <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"font-weight: bold\">]</span>, colspan: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">)</span>,\n",
       "<span style=\"font-weight: bold\">[</span>AlphaEarth<span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13.0858</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0003</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"font-weight: bold\">[</span>ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Cluster Selection<span style=\"font-weight: bold\">)]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.5899</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0594</span><span style=\"font-weight: bold\">]</span>,\n",
       "<span style=\"font-weight: bold\">[</span>ERA5 + ArcticDEM <span style=\"font-weight: bold\">(</span>Univariate Selection<span style=\"font-weight: bold\">)]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5.5879</span><span style=\"font-weight: bold\">]</span>, <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0189</span><span style=\"font-weight: bold\">]</span>,\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtable.cell\u001b[0m\u001b[1m(\u001b[0mstrong\u001b[1m[\u001b[0mH3 Level \u001b[1;36m3\u001b[0m\u001b[1m]\u001b[0m, colspan: \u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m,\n",
       "\u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m1.9745\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.1613\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1m[\u001b[0mERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.7780\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.3788\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1m[\u001b[0mERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.2241\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.6365\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1;35mtable.cell\u001b[0m\u001b[1m(\u001b[0mstrong\u001b[1m[\u001b[0mH3 Level \u001b[1;36m4\u001b[0m\u001b[1m]\u001b[0m, colspan: \u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m,\n",
       "\u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m1.1320\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.2884\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1m[\u001b[0mERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m2.6378\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.1062\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1m[\u001b[0mERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.7497\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.3878\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1;35mtable.cell\u001b[0m\u001b[1m(\u001b[0mstrong\u001b[1m[\u001b[0mH3 Level \u001b[1;36m5\u001b[0m\u001b[1m]\u001b[0m, colspan: \u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m,\n",
       "\u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m13.0858\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.0003\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1m[\u001b[0mERA5 + ArcticDEM \u001b[1m(\u001b[0mCluster Selection\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m3.5899\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.0594\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\u001b[1m[\u001b[0mERA5 + ArcticDEM \u001b[1m(\u001b[0mUnivariate Selection\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m5.5879\u001b[0m\u001b[1m]\u001b[0m, \u001b[1m[\u001b[0m\u001b[1;36m0.0189\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculate ANOVA for each grid-level and experiment, with \"ERA5 + ArcticDEM\" is the baseline\n",
    "typst_table = \"\"\n",
    "for grid_level in feature_importances_stability_filtered[\"grid_level\"].unique():\n",
    "    print(f\"Grid Level: {grid_level}\")\n",
    "    typst_table += f\"table.cell(strong[{grid_level}], colspan: 3),\\n\"\n",
    "    fis_gl = feature_importances_stability_filtered[feature_importances_stability_filtered[\"grid_level\"] == grid_level]\n",
    "    baseline = fis_gl[fis_gl[\"experiment\"] == \"ERA5 + ArcticDEM\"][\"importance_stability\"].dropna()\n",
    "    groups = []\n",
    "    for experiment in feature_importances_stability_filtered[\"experiment\"].unique():\n",
    "        if experiment == \"ERA5 + ArcticDEM\":\n",
    "            continue\n",
    "        group = fis_gl[fis_gl[\"experiment\"] == experiment][\"importance_stability\"].dropna()\n",
    "        groups.append(group)\n",
    "        stat, p_value = f_oneway(baseline, group)\n",
    "        stat_critical = 2.60  # for alpha=0.05 and df1=1 and df2=inf\n",
    "        if (p_value < 0.05) and (stat > stat_critical):\n",
    "            print(f\"  Experiment: {experiment}, ANOVA F-statistic: {stat:.4f}, p-value: {p_value:.4f} [Significant]\")\n",
    "        else:\n",
    "            print(f\"  Experiment: {experiment}, ANOVA F-statistic: {stat:.4f}, p-value: {p_value:.4f}\")\n",
    "        typst_table += f\"[{experiment}], [{stat:.4f}], [{p_value:.4f}],\\n\"\n",
    "print(\"\\nTypst Table:\\n\")\n",
    "print(typst_table)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0aba967",
   "metadata": {},
   "source": [
    "## eSPA feature weight stability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "094626a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "30b511b2",
   "metadata": {},
   "source": [
    "## Feature importance spread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9624b56b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3963457/932376112.py:6: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
      "/tmp/ipykernel_3963457/932376112.py:6: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn.axisgrid.FacetGrid\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f389d2ac550\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 107868 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m1404.\u001b[0m\u001b[39m25x600 with \u001b[0m\u001b[1;36m2\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.catplot(\n",
    "    data=feature_importances,\n",
    "    x=\"grid_level\",\n",
    "    y=\"importance\",\n",
    "    hue=\"model_type\",\n",
    "    col=\"task\",\n",
    "    dodge=True,\n",
    "    size=3,\n",
    "    height=6,\n",
    "    palette=\"cubehelix\",\n",
    ")\n",
    "# Rotate x-axis labels\n",
    "for ax in g.axes.flat:\n",
    "    ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
    "    title = ax.get_title()\n",
    "    title = title.replace(\"task = \", \"\")\n",
    "    ax.set_title(title.title())\n",
    "\n",
    "# Titleize the axes labels\n",
    "g.set_axis_labels(\"Grid\", \"Feature Importance\")\n",
    "g._legend.set_title(\"Model\")\n",
    "g.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d8592094",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>importance_std</th>\n",
       "      <th>importance_stability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Cell Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Land Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Water Area</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>X</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ERA5 + ArcticDEM</td>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>eSPA</td>\n",
       "      <td>binary</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28965</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Water Area</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28967</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>X</td>\n",
       "      <td>1.330225</td>\n",
       "      <td>0.228370</td>\n",
       "      <td>0.171678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28968</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.204674</td>\n",
       "      <td>0.169013</td>\n",
       "      <td>0.140298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28977</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings A08</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28985</th>\n",
       "      <td>AlphaEarth</td>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings A16</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2358 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "             experiment  grid_level model_type     task  \\\n",
       "\u001b[1;36m0\u001b[0m      ERA5 + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m1\u001b[0m      ERA5 + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m2\u001b[0m      ERA5 + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m4\u001b[0m      ERA5 + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[1;36m5\u001b[0m      ERA5 + ArcticDEM  H3 Level \u001b[1;36m3\u001b[0m       eSPA   binary   \n",
       "\u001b[33m...\u001b[0m                 \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m28965\u001b[0m        AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m28967\u001b[0m        AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m28968\u001b[0m        AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m28977\u001b[0m        AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\u001b[1;36m28985\u001b[0m        AlphaEarth  H3 Level \u001b[1;36m6\u001b[0m        kNN  density   \n",
       "\n",
       "                         feature  importance  importance_std  \\\n",
       "\u001b[1;36m0\u001b[0m                      Cell Area    \u001b[1;36m0.000000\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m                      Land Area    \u001b[1;36m0.000000\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m                     Water Area    \u001b[1;36m0.000000\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m                              X    \u001b[1;36m0.000000\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m5\u001b[0m                              Y    \u001b[1;36m0.000000\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[33m...\u001b[0m                          \u001b[33m...\u001b[0m         \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m28965\u001b[0m                 Water Area   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m28967\u001b[0m                          X    \u001b[1;36m1.330225\u001b[0m        \u001b[1;36m0.228370\u001b[0m   \n",
       "\u001b[1;36m28968\u001b[0m                          Y    \u001b[1;36m1.204674\u001b[0m        \u001b[1;36m0.169013\u001b[0m   \n",
       "\u001b[1;36m28977\u001b[0m  Embeddings Embeddings A08   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\u001b[1;36m28985\u001b[0m  Embeddings Embeddings A16   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m   \n",
       "\n",
       "       importance_stability  \n",
       "\u001b[1;36m0\u001b[0m                       NaN  \n",
       "\u001b[1;36m1\u001b[0m                       NaN  \n",
       "\u001b[1;36m2\u001b[0m                       NaN  \n",
       "\u001b[1;36m4\u001b[0m                       NaN  \n",
       "\u001b[1;36m5\u001b[0m                       NaN  \n",
       "\u001b[33m...\u001b[0m                     \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m28965\u001b[0m              \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m28967\u001b[0m              \u001b[1;36m0.171678\u001b[0m  \n",
       "\u001b[1;36m28968\u001b[0m              \u001b[1;36m0.140298\u001b[0m  \n",
       "\u001b[1;36m28977\u001b[0m              \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m28985\u001b[0m              \u001b[1;36m0.000000\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m2358\u001b[0m rows x \u001b[1;36m8\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_feature_importances = feature_importances.groupby([\"feature\", \"task\"])[\"importance\"].max()\n",
    "top10_feature_importances_names = (\n",
    "    max_feature_importances.reset_index()\n",
    "    .groupby(\"task\")\n",
    "    .apply(lambda df: df.sort_values(\"importance\", ascending=False).head(10), include_groups=False)[\"feature\"]  # ty:ignore[no-matching-overload]\n",
    "    .unique()\n",
    ")\n",
    "top10_feature_importances = feature_importances[feature_importances[\"feature\"].isin(top10_feature_importances_names)]\n",
    "top10_feature_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "736ab800",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn.axisgrid.FacetGrid\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7ef50fa01310\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 198764 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 1624x700 with \u001b[0m\u001b[1;36m2\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.relplot(\n",
    "    data=top10_feature_importances,\n",
    "    y=\"feature\",\n",
    "    x=\"importance\",\n",
    "    hue=\"grid_level\",\n",
    "    col=\"task\",\n",
    "    style=\"model_type\",\n",
    "    markers=True,\n",
    "    height=7,\n",
    "    aspect=1,\n",
    "    facet_kws={\"sharex\": False},\n",
    ")\n",
    "for ax in g.axes.flat:\n",
    "    title = ax.get_title()\n",
    "    title = title.replace(\"task = \", \"\")\n",
    "    ax.set_title(title.title())\n",
    "\n",
    "# As the legend has two groups instead of a title, we need to check all entries to change the titles\n",
    "for text in g._legend.texts:\n",
    "    if text.get_text() == \"grid_level\":\n",
    "        text.set_text(\"Grid\")\n",
    "    elif text.get_text() == \"model_type\":\n",
    "        text.set_text(\"\\nModel\")\n",
    "\n",
    "# Titleize the axes labels\n",
    "g.set_axis_labels(\"Feature Importance\", \"Feature\")\n",
    "g.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "75a6abf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2145397/1509558415.py:16: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
      "/tmp/ipykernel_2145397/1509558415.py:16: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 657940 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m2396.\u001b[0m\u001b[39m75x1000 with \u001b[0m\u001b[1;36m2\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.relplot(\n",
    "    data=top10_feature_importances,\n",
    "    y=\"importance\",\n",
    "    x=\"grid_level\",\n",
    "    hue=\"feature\",\n",
    "    col=\"task\",\n",
    "    style=\"model_type\",\n",
    "    markers=True,\n",
    "    height=10,\n",
    "    kind=\"line\",\n",
    "    facet_kws={\"sharey\": False},\n",
    ")\n",
    "\n",
    "\n",
    "for ax in g.axes.flat:\n",
    "    ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9881977d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "default",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
