{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "234f5da7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from rich import pretty, print, traceback\n",
    "\n",
    "from entropice.utils.paths import RESULTS_DIR\n",
    "from entropice.utils.training import Training\n",
    "\n",
    "pretty.install()\n",
    "traceback.install()\n",
    "\n",
    "sns.set_theme(context=\"paper\", style=\"whitegrid\", palette=\"cubehelix\", font_scale=1.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "073e4e89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Found <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">81</span> <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">45</span> binary, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">36</span> density<span style=\"font-weight: bold\">)</span> runs in experiment directory\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Found \u001b[1;36m81\u001b[0m \u001b[1m(\u001b[0m\u001b[1;36m45\u001b[0m binary, \u001b[1;36m36\u001b[0m density\u001b[1m)\u001b[0m runs in experiment directory\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>dataset</th>\n",
       "      <th>grid</th>\n",
       "      <th>level</th>\n",
       "      <th>members</th>\n",
       "      <th>method</th>\n",
       "      <th>task</th>\n",
       "      <th>target</th>\n",
       "      <th>model_type</th>\n",
       "      <th>run</th>\n",
       "      <th>...</th>\n",
       "      <th>test_r2</th>\n",
       "      <th>test_pearsonr</th>\n",
       "      <th>test_median_absolute_error</th>\n",
       "      <th>complete_root_mean_squared_error</th>\n",
       "      <th>complete_mean_squared_error</th>\n",
       "      <th>complete_mean_absolute_error</th>\n",
       "      <th>complete_r2</th>\n",
       "      <th>complete_pearsonr</th>\n",
       "      <th>complete_median_absolute_error</th>\n",
       "      <th>grid_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-4['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>4</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-3['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>3</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-8['AlphaEarth']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>8</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HEALPix Level 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-7['AlphaEarth']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>7</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HEALPix Level 7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-7['AlphaEarth']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>7</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HEALPix Level 7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-6['AlphaEarth']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>6</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>density</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.283430</td>\n",
       "      <td>0.587129</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000225</td>\n",
       "      <td>5.067244e-08</td>\n",
       "      <td>0.00004</td>\n",
       "      <td>0.263199</td>\n",
       "      <td>0.668917</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>HEALPix Level 6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>healpix-10['AlphaEarth']</td>\n",
       "      <td>HEALPix</td>\n",
       "      <td>10</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HEALPix Level 10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-6['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>6</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>AutoML</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-5['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>5</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>binary</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>/raid/scratch/tohoel001/data/entropice/results...</td>\n",
       "      <td>hex-6['AlphaEarth']</td>\n",
       "      <td>H3</td>\n",
       "      <td>6</td>\n",
       "      <td>[AlphaEarth]</td>\n",
       "      <td>HPOCV</td>\n",
       "      <td>density</td>\n",
       "      <td>darts_v1</td>\n",
       "      <td>kNN</td>\n",
       "      <td>Training(path=/raid/scratch/tohoel001/data/ent...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.504432</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.514065e-08</td>\n",
       "      <td>0.00001</td>\n",
       "      <td>0.912426</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H3 Level 6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>81 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "                                                 path  \\\n",
       "\u001b[1;36m0\u001b[0m   \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m1\u001b[0m   \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m2\u001b[0m   \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m3\u001b[0m   \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m4\u001b[0m   \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "..                                                \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m76\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m77\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m78\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m79\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\u001b[1;36m80\u001b[0m  \u001b[35m/raid/scratch/tohoel001/data/entropice/\u001b[0m\u001b[95mresults...\u001b[0m   \n",
       "\n",
       "                     dataset     grid  level       members  method     task  \\\n",
       "\u001b[1;36m0\u001b[0m        hex-\u001b[1;36m4\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m4\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML   binary   \n",
       "\u001b[1;36m1\u001b[0m        hex-\u001b[1;36m3\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m3\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML   binary   \n",
       "\u001b[1;36m2\u001b[0m    healpix-\u001b[1;36m8\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m  HEALPix      \u001b[1;36m8\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m   HPOCV   binary   \n",
       "\u001b[1;36m3\u001b[0m    healpix-\u001b[1;36m7\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m  HEALPix      \u001b[1;36m7\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m   HPOCV   binary   \n",
       "\u001b[1;36m4\u001b[0m    healpix-\u001b[1;36m7\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m  HEALPix      \u001b[1;36m7\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m   HPOCV   binary   \n",
       "..                       \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m           \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m76\u001b[0m   healpix-\u001b[1;36m6\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m  HEALPix      \u001b[1;36m6\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML  density   \n",
       "\u001b[1;36m77\u001b[0m  healpix-\u001b[1;36m10\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m  HEALPix     \u001b[1;36m10\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML   binary   \n",
       "\u001b[1;36m78\u001b[0m       hex-\u001b[1;36m6\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m6\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m  AutoML   binary   \n",
       "\u001b[1;36m79\u001b[0m       hex-\u001b[1;36m5\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m5\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m   HPOCV   binary   \n",
       "\u001b[1;36m80\u001b[0m       hex-\u001b[1;36m6\u001b[0m\u001b[1m[\u001b[0m\u001b[32m'AlphaEarth'\u001b[0m\u001b[1m]\u001b[0m       H3      \u001b[1;36m6\u001b[0m  \u001b[1m[\u001b[0mAlphaEarth\u001b[1m]\u001b[0m   HPOCV  density   \n",
       "\n",
       "      target     model_type  \\\n",
       "\u001b[1;36m0\u001b[0m   darts_v1      AutoGluon   \n",
       "\u001b[1;36m1\u001b[0m   darts_v1      AutoGluon   \n",
       "\u001b[1;36m2\u001b[0m   darts_v1        XGBoost   \n",
       "\u001b[1;36m3\u001b[0m   darts_v1        XGBoost   \n",
       "\u001b[1;36m4\u001b[0m   darts_v1  Random Forest   \n",
       "..       \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m76\u001b[0m  darts_v1      AutoGluon   \n",
       "\u001b[1;36m77\u001b[0m  darts_v1      AutoGluon   \n",
       "\u001b[1;36m78\u001b[0m  darts_v1      AutoGluon   \n",
       "\u001b[1;36m79\u001b[0m  darts_v1  Random Forest   \n",
       "\u001b[1;36m80\u001b[0m  darts_v1            kNN   \n",
       "\n",
       "                                                  run  \u001b[33m...\u001b[0m   test_r2  \\\n",
       "\u001b[1;36m0\u001b[0m   \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "\u001b[1;36m1\u001b[0m   \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "\u001b[1;36m2\u001b[0m   \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "\u001b[1;36m3\u001b[0m   \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "\u001b[1;36m4\u001b[0m   \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "..                                                \u001b[33m...\u001b[0m  \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m76\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m  \u001b[1;36m0.283430\u001b[0m   \n",
       "\u001b[1;36m77\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "\u001b[1;36m78\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "\u001b[1;36m79\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m       NaN   \n",
       "\u001b[1;36m80\u001b[0m  \u001b[1;35mTraining\u001b[0m\u001b[1m(\u001b[0m\u001b[33mpath\u001b[0m=\u001b[35m/raid/scratch/tohoel001/data/\u001b[0m\u001b[95ment...\u001b[0m  \u001b[33m...\u001b[0m  \u001b[1;36m0.504432\u001b[0m   \n",
       "\n",
       "    test_pearsonr  test_median_absolute_error  \\\n",
       "\u001b[1;36m0\u001b[0m             NaN                         NaN   \n",
       "\u001b[1;36m1\u001b[0m             NaN                         NaN   \n",
       "\u001b[1;36m2\u001b[0m             NaN                         NaN   \n",
       "\u001b[1;36m3\u001b[0m             NaN                         NaN   \n",
       "\u001b[1;36m4\u001b[0m             NaN                         NaN   \n",
       "..            \u001b[33m...\u001b[0m                         \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m76\u001b[0m       \u001b[1;36m0.587129\u001b[0m                    \u001b[1;36m0.000009\u001b[0m   \n",
       "\u001b[1;36m77\u001b[0m            NaN                         NaN   \n",
       "\u001b[1;36m78\u001b[0m            NaN                         NaN   \n",
       "\u001b[1;36m79\u001b[0m            NaN                         NaN   \n",
       "\u001b[1;36m80\u001b[0m            NaN                         NaN   \n",
       "\n",
       "    complete_root_mean_squared_error  complete_mean_squared_error  \\\n",
       "\u001b[1;36m0\u001b[0m                                NaN                          NaN   \n",
       "\u001b[1;36m1\u001b[0m                                NaN                          NaN   \n",
       "\u001b[1;36m2\u001b[0m                                NaN                          NaN   \n",
       "\u001b[1;36m3\u001b[0m                                NaN                          NaN   \n",
       "\u001b[1;36m4\u001b[0m                                NaN                          NaN   \n",
       "..                               \u001b[33m...\u001b[0m                          \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m76\u001b[0m                          \u001b[1;36m0.000225\u001b[0m                 \u001b[1;36m5.067244e-08\u001b[0m   \n",
       "\u001b[1;36m77\u001b[0m                               NaN                          NaN   \n",
       "\u001b[1;36m78\u001b[0m                               NaN                          NaN   \n",
       "\u001b[1;36m79\u001b[0m                               NaN                          NaN   \n",
       "\u001b[1;36m80\u001b[0m                               NaN                 \u001b[1;36m3.514065e-08\u001b[0m   \n",
       "\n",
       "    complete_mean_absolute_error  complete_r2  complete_pearsonr  \\\n",
       "\u001b[1;36m0\u001b[0m                            NaN          NaN                NaN   \n",
       "\u001b[1;36m1\u001b[0m                            NaN          NaN                NaN   \n",
       "\u001b[1;36m2\u001b[0m                            NaN          NaN                NaN   \n",
       "\u001b[1;36m3\u001b[0m                            NaN          NaN                NaN   \n",
       "\u001b[1;36m4\u001b[0m                            NaN          NaN                NaN   \n",
       "..                           \u001b[33m...\u001b[0m          \u001b[33m...\u001b[0m                \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m76\u001b[0m                       \u001b[1;36m0.00004\u001b[0m     \u001b[1;36m0.263199\u001b[0m           \u001b[1;36m0.668917\u001b[0m   \n",
       "\u001b[1;36m77\u001b[0m                           NaN          NaN                NaN   \n",
       "\u001b[1;36m78\u001b[0m                           NaN          NaN                NaN   \n",
       "\u001b[1;36m79\u001b[0m                           NaN          NaN                NaN   \n",
       "\u001b[1;36m80\u001b[0m                       \u001b[1;36m0.00001\u001b[0m     \u001b[1;36m0.912426\u001b[0m                NaN   \n",
       "\n",
       "    complete_median_absolute_error        grid_level  \n",
       "\u001b[1;36m0\u001b[0m                              NaN        H3 Level \u001b[1;36m4\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m                              NaN        H3 Level \u001b[1;36m3\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m                              NaN   HEALPix Level \u001b[1;36m8\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m                              NaN   HEALPix Level \u001b[1;36m7\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m                              NaN   HEALPix Level \u001b[1;36m7\u001b[0m  \n",
       "..                             \u001b[33m...\u001b[0m               \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m76\u001b[0m                        \u001b[1;36m0.000008\u001b[0m   HEALPix Level \u001b[1;36m6\u001b[0m  \n",
       "\u001b[1;36m77\u001b[0m                             NaN  HEALPix Level \u001b[1;36m10\u001b[0m  \n",
       "\u001b[1;36m78\u001b[0m                             NaN        H3 Level \u001b[1;36m6\u001b[0m  \n",
       "\u001b[1;36m79\u001b[0m                             NaN        H3 Level \u001b[1;36m5\u001b[0m  \n",
       "\u001b[1;36m80\u001b[0m                             NaN        H3 Level \u001b[1;36m6\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m81\u001b[0m rows x \u001b[1;36m101\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# experiment_name = \"feature_importance_era5-shoulder_arcticdem-v2\"\n",
    "# experiment_name = \"feature_importance_era5-shoulder_arcticdem-v2_selection='univariate'\"\n",
    "# experiment_name = \"feature_importance_era5-shoulder_arcticdem-v2_selection='cluster'\"\n",
    "experiment_name = \"feature_importance_alphaearth\"\n",
    "experiment_dir = RESULTS_DIR / experiment_name\n",
    "\n",
    "all_runs = pd.DataFrame(\n",
    "    [Training.load(f).to_record() for f in experiment_dir.iterdir() if f.is_dir() and f.name.startswith(\"permafrost_\")]\n",
    ")\n",
    "\n",
    "# Rename the models: \"rf\" to \"Random Forest\", \"xgboost\" to \"XGBoost\", \"knn\" to \"kNN\",\n",
    "# \"espa\" to \"eSPA\" and \"autogluon\" to \"AutoGluon for better readability\n",
    "all_runs[\"model_type\"] = (\n",
    "    all_runs[\"model_type\"]\n",
    "    .str.replace(\"rf\", \"Random Forest\", regex=False)\n",
    "    .str.replace(\"xgboost\", \"XGBoost\", regex=False)\n",
    "    .str.replace(\"knn\", \"kNN\", regex=False)\n",
    "    .str.replace(\"espa\", \"eSPA\", regex=False)\n",
    "    .str.replace(\"autogluon\", \"AutoGluon\", regex=False)\n",
    ")\n",
    "\n",
    "# Rename hex to H3 and healpix to HEALPix for better readability\n",
    "all_runs[\"grid\"] = all_runs[\"grid\"].str.replace(\"hex\", \"H3\", regex=False).str.replace(\"healpix\", \"HEALPix\", regex=False)\n",
    "\n",
    "# Create a combined \"grid_level\" column for easier plotting\n",
    "grid_level = all_runs[\"grid\"].str.cat(all_runs[\"level\"].astype(str), sep=\" Level \")\n",
    "# Make it categorical with a specific order\n",
    "grid_level_cat = pd.CategoricalDtype(\n",
    "    categories=[\n",
    "        \"H3 Level 3\",\n",
    "        \"HEALPix Level 6\",\n",
    "        \"HEALPix Level 7\",\n",
    "        \"H3 Level 4\",\n",
    "        \"HEALPix Level 8\",\n",
    "        \"H3 Level 5\",\n",
    "        \"HEALPix Level 9\",\n",
    "        \"HEALPix Level 10\",\n",
    "        \"H3 Level 6\",\n",
    "    ],\n",
    "    ordered=True,\n",
    ")\n",
    "grid_level = grid_level.astype(grid_level_cat)\n",
    "all_runs[\"grid_level\"] = grid_level\n",
    "\n",
    "binary_runs = all_runs[all_runs[\"task\"] == \"binary\"]\n",
    "density_runs = all_runs[all_runs[\"task\"] == \"density\"]\n",
    "# Drop all columns that are all NaN in either binary_runs or density_runs\n",
    "binary_runs = binary_runs.dropna(axis=1, how=\"all\")\n",
    "density_runs = density_runs.dropna(axis=1, how=\"all\")\n",
    "\n",
    "print(f\"Found {len(all_runs)} ({len(binary_runs)} binary, {len(density_runs)} density) runs in experiment directory\")\n",
    "all_runs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cbbc1c8",
   "metadata": {},
   "source": [
    "## Model performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "4dcec91f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>score</th>\n",
       "      <th>split</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.950962</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>H3 Level 3</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.973958</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HEALPix Level 7</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.993769</td>\n",
       "      <td>train</td>\n",
       "      <td>f1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.501394</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.824979</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.515731</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.510555</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.903151</td>\n",
       "      <td>test</td>\n",
       "      <td>recall</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "           grid_level     model_type     score  split  metric\n",
       "\u001b[1;36m0\u001b[0m          H3 Level \u001b[1;36m4\u001b[0m      AutoGluon  \u001b[1;36m0.950962\u001b[0m  train      f1\n",
       "\u001b[1;36m1\u001b[0m          H3 Level \u001b[1;36m3\u001b[0m      AutoGluon  \u001b[1;36m0.973958\u001b[0m  train      f1\n",
       "\u001b[1;36m2\u001b[0m     HEALPix Level \u001b[1;36m8\u001b[0m        XGBoost  \u001b[1;36m1.000000\u001b[0m  train      f1\n",
       "\u001b[1;36m3\u001b[0m     HEALPix Level \u001b[1;36m7\u001b[0m        XGBoost  \u001b[1;36m1.000000\u001b[0m  train      f1\n",
       "\u001b[1;36m4\u001b[0m     HEALPix Level \u001b[1;36m7\u001b[0m  Random Forest  \u001b[1;36m0.993769\u001b[0m  train      f1\n",
       "..                \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m     \u001b[33m...\u001b[0m\n",
       "\u001b[1;36m265\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m  Random Forest  \u001b[1;36m0.501394\u001b[0m   test  recall\n",
       "\u001b[1;36m266\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m        XGBoost  \u001b[1;36m0.824979\u001b[0m   test  recall\n",
       "\u001b[1;36m267\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m      AutoGluon  \u001b[1;36m0.515731\u001b[0m   test  recall\n",
       "\u001b[1;36m268\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m      AutoGluon  \u001b[1;36m0.510555\u001b[0m   test  recall\n",
       "\u001b[1;36m269\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m  Random Forest  \u001b[1;36m0.903151\u001b[0m   test  recall\n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m270\u001b[0m rows x \u001b[1;36m5\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bin_metrics = [\"f1\", \"precision\", \"recall\"]\n",
    "splits = [\"train\", \"test\"]\n",
    "# Create a long form DataFrame for easier plotting with seaborn\n",
    "binary_long = binary_runs.melt(\n",
    "    id_vars=[\"grid_level\", \"model_type\"],\n",
    "    value_vars=[f\"{split}_{metric}\" for split in splits for metric in bin_metrics],\n",
    "    var_name=\"split_metric\",\n",
    "    value_name=\"score\",\n",
    ")\n",
    "# Split the \"split_metric\" column into separate \"split\" and \"metric\" columns\n",
    "binary_long[[\"split\", \"metric\"]] = binary_long[\"split_metric\"].str.rsplit(\"_\", n=1, expand=True)\n",
    "# Drop the original \"split_metric\" column\n",
    "binary_long = binary_long.drop(columns=[\"split_metric\"])\n",
    "binary_long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "0e792a50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>score</th>\n",
       "      <th>split</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HEALPix Level 8</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.388279</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.933486</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>kNN</td>\n",
       "      <td>0.995971</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.927457</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.826636</td>\n",
       "      <td>train</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>HEALPix Level 9</td>\n",
       "      <td>kNN</td>\n",
       "      <td>0.600174</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>HEALPix Level 10</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.509789</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>H3 Level 5</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.642095</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>HEALPix Level 6</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>0.283430</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>0.504432</td>\n",
       "      <td>test</td>\n",
       "      <td>r2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>72 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "          grid_level     model_type     score  split metric\n",
       "\u001b[1;36m0\u001b[0m    HEALPix Level \u001b[1;36m8\u001b[0m      AutoGluon  \u001b[1;36m0.388279\u001b[0m  train     r2\n",
       "\u001b[1;36m1\u001b[0m    HEALPix Level \u001b[1;36m9\u001b[0m  Random Forest  \u001b[1;36m0.933486\u001b[0m  train     r2\n",
       "\u001b[1;36m2\u001b[0m         H3 Level \u001b[1;36m4\u001b[0m            kNN  \u001b[1;36m0.995971\u001b[0m  train     r2\n",
       "\u001b[1;36m3\u001b[0m         H3 Level \u001b[1;36m6\u001b[0m      AutoGluon  \u001b[1;36m0.927457\u001b[0m  train     r2\n",
       "\u001b[1;36m4\u001b[0m         H3 Level \u001b[1;36m4\u001b[0m      AutoGluon  \u001b[1;36m0.826636\u001b[0m  train     r2\n",
       "..               \u001b[33m...\u001b[0m            \u001b[33m...\u001b[0m       \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m    \u001b[33m...\u001b[0m\n",
       "\u001b[1;36m67\u001b[0m   HEALPix Level \u001b[1;36m9\u001b[0m            kNN  \u001b[1;36m0.600174\u001b[0m   test     r2\n",
       "\u001b[1;36m68\u001b[0m  HEALPix Level \u001b[1;36m10\u001b[0m        XGBoost  \u001b[1;36m0.509789\u001b[0m   test     r2\n",
       "\u001b[1;36m69\u001b[0m        H3 Level \u001b[1;36m5\u001b[0m  Random Forest  \u001b[1;36m0.642095\u001b[0m   test     r2\n",
       "\u001b[1;36m70\u001b[0m   HEALPix Level \u001b[1;36m6\u001b[0m      AutoGluon  \u001b[1;36m0.283430\u001b[0m   test     r2\n",
       "\u001b[1;36m71\u001b[0m        H3 Level \u001b[1;36m6\u001b[0m            kNN  \u001b[1;36m0.504432\u001b[0m   test     r2\n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m72\u001b[0m rows x \u001b[1;36m5\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "density_metrics = [\"r2\"]\n",
    "# Create a long form DataFrame for easier plotting with seaborn\n",
    "density_long = density_runs.melt(\n",
    "    id_vars=[\"grid_level\", \"model_type\"],\n",
    "    value_vars=[f\"{split}_{metric}\" for split in splits for metric in density_metrics],\n",
    "    var_name=\"split_metric\",\n",
    "    value_name=\"score\",\n",
    ")\n",
    "# Split the \"split_metric\" column into separate \"split\" and \"metric\" columns\n",
    "density_long[[\"split\", \"metric\"]] = density_long[\"split_metric\"].str.split(\"_\", n=1, expand=True)\n",
    "# Drop the original \"split_metric\" column\n",
    "density_long = density_long.drop(columns=[\"split_metric\"])\n",
    "density_long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "d200d56d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn.axisgrid.FacetGrid\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f44724cc2d0\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 294116 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m2204.\u001b[0m\u001b[39m25x1000 with \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.relplot(\n",
    "    pd.concat([binary_long, density_long]),\n",
    "    x=\"grid_level\",\n",
    "    y=\"score\",\n",
    "    col=\"metric\",\n",
    "    row=\"split\",\n",
    "    hue=\"model_type\",\n",
    "    kind=\"line\",\n",
    "    style=\"model_type\",\n",
    "    hue_order=reversed([\"AutoGluon\", \"XGBoost\", \"Random Forest\", \"kNN\", \"eSPA\"]),\n",
    "    style_order=reversed([\"AutoGluon\", \"XGBoost\", \"Random Forest\", \"kNN\", \"eSPA\"]),\n",
    "    markers=True,\n",
    "    height=5,\n",
    ")\n",
    "for ax in g.axes.flat:\n",
    "    for label in ax.get_xticklabels():\n",
    "        # Align x-axis labels right and rotate them by 45 degrees\n",
    "        label.set_horizontalalignment(\"right\")\n",
    "        label.set_rotation(45)\n",
    "    # Titleize the axes title\n",
    "    title = ax.get_title()\n",
    "    title = (\n",
    "        title.replace(\"metric = \", \"\")\n",
    "        .replace(\"split = \", \"\")\n",
    "        .replace(\"train\", \"train-split\")\n",
    "        .replace(\"test\", \"test-split\")\n",
    "    )\n",
    "    ax.set_title(title.title())\n",
    "\n",
    "# Change the legend title to \"Model\"\n",
    "g._legend.set_title(\"Model\")\n",
    "\n",
    "# Change all the titles to be more descriptive\n",
    "g.set_axis_labels(\"Grid\", \"Score\")\n",
    "g.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceabcdf5",
   "metadata": {},
   "source": [
    "## Most important features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "b062bccf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>importance_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A60</td>\n",
       "      <td>0.010345</td>\n",
       "      <td>0.005122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A46</td>\n",
       "      <td>0.009360</td>\n",
       "      <td>0.002401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>X</td>\n",
       "      <td>0.007635</td>\n",
       "      <td>0.001827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A53</td>\n",
       "      <td>0.006404</td>\n",
       "      <td>0.003066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A00</td>\n",
       "      <td>0.002956</td>\n",
       "      <td>0.002061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10849</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A59</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10850</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A60</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10851</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A61</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10852</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A62</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10853</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A63</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10854 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "       grid_level model_type     task                          feature  \\\n",
       "\u001b[1;36m0\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A60   \n",
       "\u001b[1;36m1\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A46   \n",
       "\u001b[1;36m2\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary                                X   \n",
       "\u001b[1;36m3\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A53   \n",
       "\u001b[1;36m4\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A00   \n",
       "\u001b[33m...\u001b[0m           \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m                              \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m10849\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A59   \n",
       "\u001b[1;36m10850\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A60   \n",
       "\u001b[1;36m10851\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A61   \n",
       "\u001b[1;36m10852\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A62   \n",
       "\u001b[1;36m10853\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A63   \n",
       "\n",
       "       importance  importance_std  \n",
       "\u001b[1;36m0\u001b[0m        \u001b[1;36m0.010345\u001b[0m        \u001b[1;36m0.005122\u001b[0m  \n",
       "\u001b[1;36m1\u001b[0m        \u001b[1;36m0.009360\u001b[0m        \u001b[1;36m0.002401\u001b[0m  \n",
       "\u001b[1;36m2\u001b[0m        \u001b[1;36m0.007635\u001b[0m        \u001b[1;36m0.001827\u001b[0m  \n",
       "\u001b[1;36m3\u001b[0m        \u001b[1;36m0.006404\u001b[0m        \u001b[1;36m0.003066\u001b[0m  \n",
       "\u001b[1;36m4\u001b[0m        \u001b[1;36m0.002956\u001b[0m        \u001b[1;36m0.002061\u001b[0m  \n",
       "\u001b[33m...\u001b[0m           \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m10849\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10850\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10851\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10852\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10853\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m10854\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create dataframe in long format for the feature importances\n",
    "# Cols: grid_level, model_type, task, feature, importance, stddev\n",
    "feature_importances = []\n",
    "for run in all_runs[\"run\"]:\n",
    "    task = run.task\n",
    "    grid = run.dataset.grid.replace(\"hex\", \"H3\").replace(\"healpix\", \"HEALPix\")\n",
    "    grid_level = f\"{grid} Level {run.dataset.level}\"\n",
    "    model_type = (\n",
    "        run.model_type.replace(\"rf\", \"Random Forest\")\n",
    "        .replace(\"xgboost\", \"XGBoost\")\n",
    "        .replace(\"knn\", \"kNN\")\n",
    "        .replace(\"espa\", \"eSPA\")\n",
    "        .replace(\"autogluon\", \"AutoGluon\")\n",
    "    )\n",
    "    importances = run.feature_importance[\"importance\"]\n",
    "    importances_std = run.feature_importance[\"stddev\"]\n",
    "    for feature, importance in importances.items():\n",
    "        feature_importances.append(\n",
    "            {\n",
    "                \"grid_level\": grid_level,\n",
    "                \"model_type\": model_type,\n",
    "                \"task\": task,\n",
    "                \"feature\": feature,\n",
    "                \"importance\": importance,\n",
    "                \"importance_std\": importances_std.get(feature, 0.0),\n",
    "            }\n",
    "        )\n",
    "feature_importances = pd.DataFrame(feature_importances)\n",
    "feature_importances[\"grid_level\"] = feature_importances[\"grid_level\"].astype(grid_level_cat)\n",
    "\n",
    "\n",
    "def _parse_feature_name(feature_name: str) -> str:\n",
    "    feature_name = (\n",
    "        feature_name.replace(\"_median\", \"_\")\n",
    "        .replace(\"_mean\", \"_\")\n",
    "        .replace(\"era5_\", \"\")\n",
    "        .replace(\"arcticdem_\", \"\")\n",
    "        .replace(\"JFM\", \"Winter\")\n",
    "        .replace(\"AMJ\", \"Spring\")\n",
    "        .replace(\"JAS\", \"Summer\")\n",
    "        .replace(\"OND\", \"Fall\")\n",
    "        .replace(\"_\", \" \")\n",
    "        .title()\n",
    "    )\n",
    "    # Remove duplicate spaces\n",
    "    feature_name = \" \".join(feature_name.split())\n",
    "    return feature_name\n",
    "\n",
    "\n",
    "feature_importances[\"feature\"] = feature_importances[\"feature\"].apply(_parse_feature_name)\n",
    "\n",
    "feature_importances_binary = feature_importances[feature_importances[\"task\"] == \"binary\"]\n",
    "feature_importances_density = feature_importances[feature_importances[\"task\"] == \"density\"]\n",
    "\n",
    "feature_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "9624b56b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2145397/934192566.py:6: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
      "/tmp/ipykernel_2145397/934192566.py:6: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn.axisgrid.FacetGrid\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f44ab2f3890\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 98164 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m1404.\u001b[0m\u001b[39m25x600 with \u001b[0m\u001b[1;36m2\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.catplot(\n",
    "    data=feature_importances, x=\"grid_level\", y=\"importance\", hue=\"model_type\", col=\"task\", dodge=True, size=3, height=6\n",
    ")\n",
    "# Rotate x-axis labels\n",
    "for ax in g.axes.flat:\n",
    "    ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
    "    title = ax.get_title()\n",
    "    title = title.replace(\"task = \", \"\")\n",
    "    ax.set_title(title.title())\n",
    "\n",
    "# Titleize the axes labels\n",
    "g.set_axis_labels(\"Grid\", \"Feature Importance\")\n",
    "g._legend.set_title(\"Model\")\n",
    "g.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "aeb03cee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2145397/1002040249.py:14: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
      "/tmp/ipykernel_2145397/1002040249.py:14: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn.axisgrid.FacetGrid\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f44d3622210\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 211464 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m1404.\u001b[0m\u001b[39m25x600 with \u001b[0m\u001b[1;36m2\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.relplot(\n",
    "    feature_importances,\n",
    "    x=\"grid_level\",\n",
    "    y=\"importance_std\",\n",
    "    hue=\"model_type\",\n",
    "    col=\"task\",\n",
    "    kind=\"line\",\n",
    "    style=\"model_type\",\n",
    "    markers=True,\n",
    "    height=6,\n",
    "    facet_kws={\"sharey\": False},\n",
    ")\n",
    "for ax in g.axes.flat:\n",
    "    ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
    "    # Titleize the axes title\n",
    "    title = ax.get_title()\n",
    "    title = title.replace(\"task = \", \"\")\n",
    "    ax.set_title(title.title())\n",
    "\n",
    "\n",
    "# Titleize the axes labels\n",
    "g.set_axis_labels(\"Grid\", \"Standard Deviation of Feature Importance\")\n",
    "g._legend.set_title(\"Model\")\n",
    "g.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "d8592094",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid_level</th>\n",
       "      <th>model_type</th>\n",
       "      <th>task</th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>importance_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>X</td>\n",
       "      <td>0.007635</td>\n",
       "      <td>0.001827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A16</td>\n",
       "      <td>0.001970</td>\n",
       "      <td>0.001102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A33</td>\n",
       "      <td>0.001724</td>\n",
       "      <td>0.002237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A34</td>\n",
       "      <td>0.001232</td>\n",
       "      <td>0.000871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>H3 Level 4</td>\n",
       "      <td>AutoGluon</td>\n",
       "      <td>binary</td>\n",
       "      <td>Embeddings Embeddings A38</td>\n",
       "      <td>0.001232</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10764</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings A38</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10771</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings A45</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10794</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A04</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10799</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A09</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10835</th>\n",
       "      <td>H3 Level 6</td>\n",
       "      <td>kNN</td>\n",
       "      <td>density</td>\n",
       "      <td>Embeddings Embeddings Trend A45</td>\n",
       "      <td>-0.059563</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1296 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "\n",
       "       grid_level model_type     task                          feature  \\\n",
       "\u001b[1;36m2\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary                                X   \n",
       "\u001b[1;36m5\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A16   \n",
       "\u001b[1;36m7\u001b[0m      H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A33   \n",
       "\u001b[1;36m13\u001b[0m     H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A34   \n",
       "\u001b[1;36m15\u001b[0m     H3 Level \u001b[1;36m4\u001b[0m  AutoGluon   binary        Embeddings Embeddings A38   \n",
       "\u001b[33m...\u001b[0m           \u001b[33m...\u001b[0m        \u001b[33m...\u001b[0m      \u001b[33m...\u001b[0m                              \u001b[33m...\u001b[0m   \n",
       "\u001b[1;36m10764\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density        Embeddings Embeddings A38   \n",
       "\u001b[1;36m10771\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density        Embeddings Embeddings A45   \n",
       "\u001b[1;36m10794\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A04   \n",
       "\u001b[1;36m10799\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A09   \n",
       "\u001b[1;36m10835\u001b[0m  H3 Level \u001b[1;36m6\u001b[0m        kNN  density  Embeddings Embeddings Trend A45   \n",
       "\n",
       "       importance  importance_std  \n",
       "\u001b[1;36m2\u001b[0m        \u001b[1;36m0.007635\u001b[0m        \u001b[1;36m0.001827\u001b[0m  \n",
       "\u001b[1;36m5\u001b[0m        \u001b[1;36m0.001970\u001b[0m        \u001b[1;36m0.001102\u001b[0m  \n",
       "\u001b[1;36m7\u001b[0m        \u001b[1;36m0.001724\u001b[0m        \u001b[1;36m0.002237\u001b[0m  \n",
       "\u001b[1;36m13\u001b[0m       \u001b[1;36m0.001232\u001b[0m        \u001b[1;36m0.000871\u001b[0m  \n",
       "\u001b[1;36m15\u001b[0m       \u001b[1;36m0.001232\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[33m...\u001b[0m           \u001b[33m...\u001b[0m             \u001b[33m...\u001b[0m  \n",
       "\u001b[1;36m10764\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10771\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10794\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10799\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\u001b[1;36m10835\u001b[0m   \u001b[1;36m-0.059563\u001b[0m        \u001b[1;36m0.000000\u001b[0m  \n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[1;36m1296\u001b[0m rows x \u001b[1;36m6\u001b[0m columns\u001b[1m]\u001b[0m"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_feature_importances = feature_importances.groupby([\"feature\", \"task\"])[\"importance\"].max()\n",
    "top10_feature_importances_names = (\n",
    "    max_feature_importances.reset_index()\n",
    "    .groupby(\"task\")\n",
    "    .apply(lambda df: df.sort_values(\"importance\", ascending=False).head(10), include_groups=False)[\"feature\"]  # ty:ignore[no-matching-overload]\n",
    "    .unique()\n",
    ")\n",
    "top10_feature_importances = feature_importances[feature_importances[\"feature\"].isin(top10_feature_importances_names)]\n",
    "top10_feature_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "736ab800",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mseaborn.axisgrid.FacetGrid\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f44aadabd90\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 230064 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size 2224x1000 with \u001b[0m\u001b[1;36m2\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.relplot(\n",
    "    data=top10_feature_importances,\n",
    "    y=\"feature\",\n",
    "    x=\"importance\",\n",
    "    hue=\"grid_level\",\n",
    "    col=\"task\",\n",
    "    style=\"model_type\",\n",
    "    markers=True,\n",
    "    height=10,\n",
    "    aspect=1,\n",
    "    facet_kws={\"sharex\": False},\n",
    ")\n",
    "for ax in g.axes.flat:\n",
    "    title = ax.get_title()\n",
    "    title = title.replace(\"task = \", \"\")\n",
    "    ax.set_title(title.title())\n",
    "\n",
    "# As the legend has two groups instead of a title, we need to check all entries to change the titles\n",
    "for text in g._legend.texts:\n",
    "    if text.get_text() == \"grid_level\":\n",
    "        text.set_text(\"Grid\")\n",
    "    elif text.get_text() == \"model_type\":\n",
    "        text.set_text(\"\\nModel\")\n",
    "\n",
    "# Titleize the axes labels\n",
    "g.set_axis_labels(\"Feature Importance\", \"Feature\")\n",
    "g.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "75a6abf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2145397/1509558415.py:16: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
      "/tmp/ipykernel_2145397/1509558415.py:16: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 657940 characters]",
      "text/plain": [
       "\u001b[1m<\u001b[0m\u001b[1;95mFigure\u001b[0m\u001b[39m size \u001b[0m\u001b[1;36m2396.\u001b[0m\u001b[39m75x1000 with \u001b[0m\u001b[1;36m2\u001b[0m\u001b[39m Axes\u001b[0m\u001b[1m>\u001b[0m"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "g = sns.relplot(\n",
    "    data=top10_feature_importances,\n",
    "    y=\"importance\",\n",
    "    x=\"grid_level\",\n",
    "    hue=\"feature\",\n",
    "    col=\"task\",\n",
    "    style=\"model_type\",\n",
    "    markers=True,\n",
    "    height=10,\n",
    "    kind=\"line\",\n",
    "    facet_kws={\"sharey\": False},\n",
    ")\n",
    "\n",
    "\n",
    "for ax in g.axes.flat:\n",
    "    ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha=\"right\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9881977d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "default",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
