{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extreme Gradient Boosting (XGBoost) Training and Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import os\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "import seaborn as sns\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Data from Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connect to the database\n",
    "conn = sqlite3.connect('../features.db')\n",
    "c = conn.cursor()\n",
    "# get training, validation and test data\n",
    "train = pd.read_sql_query(\"SELECT * FROM train\", conn)\n",
    "valid = pd.read_sql_query(\"SELECT * FROM validation\", conn)\n",
    "test = pd.read_sql_query(\"SELECT * FROM test\", conn)\n",
    "# close the connection\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Format Data for Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x shape: (3502, 10)\n",
      "test_x shape: (438, 10)\n",
      "valid_x shape: (438, 10)\n",
      "features: ['age', 'gender', 'artial_rate', 'ventricular_rate', 'qrs_duration', 'qt_length', 'qrs_count', 'q_peak', 'r_axis', 't_axis']\n",
      "number of classes: 4\n"
     ]
    }
   ],
   "source": [
    "# get the target and features\n",
    "train_y = train['y']\n",
    "train_y = train_y.map({'GSVT': 0, 'AFIB': 1, 'SR': 2, 'SB': 3})\n",
    "train_x = train.drop(columns=['y'])\n",
    "\n",
    "valid_y = valid['y']\n",
    "valid_y = valid_y.map({'GSVT': 0, 'AFIB': 1, 'SR': 2, 'SB': 3})\n",
    "valid_x = valid.drop(columns=['y'])\n",
    "\n",
    "test_y = test['y']\n",
    "test_y = test_y.map({'GSVT': 0, 'AFIB': 1, 'SR': 2, 'SB': 3})\n",
    "test_x = test.drop(columns=['y'])\n",
    "\n",
    "# drop id column\n",
    "train_x = train_x.drop(columns=['id'])\n",
    "valid_x = valid_x.drop(columns=['id'])\n",
    "test_x = test_x.drop(columns=['id'])\n",
    "\n",
    "print('train_x shape:', train_x.shape)\n",
    "print('test_x shape:', test_x.shape)\n",
    "print('valid_x shape:', valid_x.shape)\n",
    "\n",
    "# print column names\n",
    "print('features:', train_x.columns.to_list())\n",
    "\n",
    "# use xgboost\n",
    "dtrain = xgb.DMatrix(train_x, label=train_y)\n",
    "dvalid = xgb.DMatrix(valid_x, label=valid_y)\n",
    "dtest = xgb.DMatrix(test_x, label=test_y)\n",
    "\n",
    "num_classes= len(set(valid_y.to_list()))\n",
    "print('number of classes:', num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Grid for Hyperparameter Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {\n",
    "    'max_depth': [3, 4, 5],\n",
    "    'min_child_weight': [1, 2, 3],\n",
    "    'eta': [0.1, 0.2, 0.3],\n",
    "    'learning_rate': [0.1, 0.2, 0.3],\n",
    "    'n_estimators': [100, 200, 300]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a XGBClassifier object\n",
    "model = xgb.XGBClassifier(objective='multi:softmax', num_class=num_classes, eval_metric='merror')\n",
    "\n",
    "# Create the grid search object\n",
    "grid_search = GridSearchCV(model, param_grid, cv=3, scoring='accuracy')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 2h 15min 58s\n",
      "Wall time: 10min\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=3,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False,\n",
       "                                     eval_metric=&#x27;merror&#x27;, gamma=None,\n",
       "                                     gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None, max_bin=None,\n",
       "                                     ma...\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None, num_class=4,\n",
       "                                     num_parallel_tree=None,\n",
       "                                     objective=&#x27;multi:softmax&#x27;, predictor=None,\n",
       "                                     random_state=None, ...),\n",
       "             param_grid={&#x27;eta&#x27;: [0.1, 0.2, 0.3],\n",
       "                         &#x27;learning_rate&#x27;: [0.1, 0.2, 0.3],\n",
       "                         &#x27;max_depth&#x27;: [3, 4, 5], &#x27;min_child_weight&#x27;: [1, 2, 3],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300]},\n",
       "             scoring=&#x27;accuracy&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=3,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False,\n",
       "                                     eval_metric=&#x27;merror&#x27;, gamma=None,\n",
       "                                     gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None, max_bin=None,\n",
       "                                     ma...\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None, num_class=4,\n",
       "                                     num_parallel_tree=None,\n",
       "                                     objective=&#x27;multi:softmax&#x27;, predictor=None,\n",
       "                                     random_state=None, ...),\n",
       "             param_grid={&#x27;eta&#x27;: [0.1, 0.2, 0.3],\n",
       "                         &#x27;learning_rate&#x27;: [0.1, 0.2, 0.3],\n",
       "                         &#x27;max_depth&#x27;: [3, 4, 5], &#x27;min_child_weight&#x27;: [1, 2, 3],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300]},\n",
       "             scoring=&#x27;accuracy&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=&#x27;merror&#x27;, gamma=None,\n",
       "              gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
       "              max_leaves=None, min_child_weight=None, missing=nan,\n",
       "              monotone_constraints=None, n_estimators=100, n_jobs=None,\n",
       "              num_class=4, num_parallel_tree=None, objective=&#x27;multi:softmax&#x27;,\n",
       "              predictor=None, random_state=None, ...)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=&#x27;merror&#x27;, gamma=None,\n",
       "              gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
       "              max_leaves=None, min_child_weight=None, missing=nan,\n",
       "              monotone_constraints=None, n_estimators=100, n_jobs=None,\n",
       "              num_class=4, num_parallel_tree=None, objective=&#x27;multi:softmax&#x27;,\n",
       "              predictor=None, random_state=None, ...)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=3,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False,\n",
       "                                     eval_metric='merror', gamma=None,\n",
       "                                     gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None, max_bin=None,\n",
       "                                     ma...\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None, num_class=4,\n",
       "                                     num_parallel_tree=None,\n",
       "                                     objective='multi:softmax', predictor=None,\n",
       "                                     random_state=None, ...),\n",
       "             param_grid={'eta': [0.1, 0.2, 0.3],\n",
       "                         'learning_rate': [0.1, 0.2, 0.3],\n",
       "                         'max_depth': [3, 4, 5], 'min_child_weight': [1, 2, 3],\n",
       "                         'n_estimators': [100, 200, 300]},\n",
       "             scoring='accuracy')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Fit the grid search object to the data\n",
    "grid_search.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters: {'eta': 0.1, 'learning_rate': 0.1, 'max_depth': 5, 'min_child_weight': 3, 'n_estimators': 100}\n",
      "Best score: 0.8012537024646579\n"
     ]
    }
   ],
   "source": [
    "# Print the best parameters and the best score\n",
    "print(f'Best parameters: {grid_search.best_params_}')\n",
    "print(f'Best score: {grid_search.best_score_}')\n",
    "#{'eta': 0.1, 'learning_rate': 0.1, 'max_depth': 5, 'min_child_weight': 3, 'n_estimators': 100}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the best model\n",
    "best_model = grid_search.best_estimator_\n",
    "# timestamp\n",
    "timestamp = datetime.now().strftime('%Y%m%d%H%M%S')\n",
    "best_model.save_model(f'../ml_models/best_xgb_model_{timestamp}.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example Training of best Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "load the best model to get the best hyperparameters from it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model: {'best_iteration': '99', 'best_ntree_limit': '100', 'scikit_learn': '{\"use_label_encoder\": false, \"n_estimators\": 100, \"objective\": \"multi:softmax\", \"max_depth\": 5, \"max_leaves\": null, \"max_bin\": null, \"grow_policy\": null, \"learning_rate\": 0.1, \"verbosity\": null, \"booster\": null, \"tree_method\": null, \"gamma\": null, \"min_child_weight\": 3, \"max_delta_step\": null, \"subsample\": null, \"sampling_method\": null, \"colsample_bytree\": null, \"colsample_bylevel\": null, \"colsample_bynode\": null, \"reg_alpha\": null, \"reg_lambda\": null, \"scale_pos_weight\": null, \"base_score\": null, \"missing\": NaN, \"num_parallel_tree\": null, \"random_state\": null, \"n_jobs\": null, \"monotone_constraints\": null, \"interaction_constraints\": null, \"importance_type\": null, \"gpu_id\": null, \"validate_parameters\": null, \"predictor\": null, \"enable_categorical\": false, \"max_cat_to_onehot\": null, \"eval_metric\": \"merror\", \"early_stopping_rounds\": null, \"callbacks\": null, \"kwargs\": {\"num_class\": 4, \"eta\": 0.1}, \"classes_\": [0, 1, 2, 3], \"n_classes_\": 4, \"_estimator_type\": \"classifier\"}'}\n"
     ]
    }
   ],
   "source": [
    "# list directory\n",
    "models = os.listdir('../ml_models')\n",
    "model_path = [model for model in models if 'json' in model and 'best' in model and 'xgb' in model][0]\n",
    "model_path = f'../ml_models/{model_path}'\n",
    "# load the best model\n",
    "best_model = xgb.Booster()\n",
    "best_model.load_model(model_path)\n",
    "best_params = best_model.attributes()\n",
    "print('best model:', best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[16:58:49] WARNING: C:/Users/administrator/workspace/xgboost-win64_release_1.6.0/src/learner.cc:627: \n",
      "Parameters: { \"best_iteration\", \"best_ntree_limit\", \"scikit_learn\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "[0]\ttrain-merror:0.16762\teval-merror:0.22603\n",
      "[1]\ttrain-merror:0.15220\teval-merror:0.22374\n",
      "[2]\ttrain-merror:0.13849\teval-merror:0.21461\n",
      "[3]\ttrain-merror:0.13535\teval-merror:0.20776\n",
      "[4]\ttrain-merror:0.13278\teval-merror:0.20091\n",
      "[5]\ttrain-merror:0.12907\teval-merror:0.20548\n",
      "[6]\ttrain-merror:0.12307\teval-merror:0.20320\n",
      "[7]\ttrain-merror:0.11850\teval-merror:0.20320\n",
      "[8]\ttrain-merror:0.11422\teval-merror:0.19406\n",
      "[9]\ttrain-merror:0.10965\teval-merror:0.20091\n",
      "[10]\ttrain-merror:0.10280\teval-merror:0.20320\n",
      "[11]\ttrain-merror:0.09880\teval-merror:0.19406\n",
      "[12]\ttrain-merror:0.09423\teval-merror:0.19406\n",
      "[13]\ttrain-merror:0.09109\teval-merror:0.19863\n",
      "[14]\ttrain-merror:0.08709\teval-merror:0.19863\n",
      "[15]\ttrain-merror:0.08195\teval-merror:0.19863\n",
      "[16]\ttrain-merror:0.07910\teval-merror:0.20091\n",
      "[17]\ttrain-merror:0.07624\teval-merror:0.19635\n",
      "[18]\ttrain-merror:0.06967\teval-merror:0.19863\n",
      "[19]\ttrain-merror:0.06710\teval-merror:0.19406\n",
      "[20]\ttrain-merror:0.06254\teval-merror:0.19178\n",
      "[21]\ttrain-merror:0.06025\teval-merror:0.19863\n",
      "[22]\ttrain-merror:0.05682\teval-merror:0.20091\n",
      "[23]\ttrain-merror:0.05311\teval-merror:0.20091\n",
      "[24]\ttrain-merror:0.05168\teval-merror:0.20320\n",
      "[25]\ttrain-merror:0.04940\teval-merror:0.19406\n",
      "[26]\ttrain-merror:0.04597\teval-merror:0.20091\n",
      "[27]\ttrain-merror:0.04397\teval-merror:0.19863\n",
      "[28]\ttrain-merror:0.04112\teval-merror:0.19863\n",
      "[29]\ttrain-merror:0.04026\teval-merror:0.19863\n",
      "[30]\ttrain-merror:0.03769\teval-merror:0.19635\n",
      "[31]\ttrain-merror:0.03712\teval-merror:0.20091\n",
      "[32]\ttrain-merror:0.03626\teval-merror:0.20320\n",
      "[33]\ttrain-merror:0.03541\teval-merror:0.20091\n",
      "[34]\ttrain-merror:0.03370\teval-merror:0.19863\n",
      "[35]\ttrain-merror:0.03113\teval-merror:0.19635\n",
      "[36]\ttrain-merror:0.02970\teval-merror:0.19635\n",
      "[37]\ttrain-merror:0.02798\teval-merror:0.19406\n",
      "[38]\ttrain-merror:0.02713\teval-merror:0.19178\n",
      "[39]\ttrain-merror:0.02513\teval-merror:0.18950\n",
      "[40]\ttrain-merror:0.02370\teval-merror:0.19178\n",
      "[41]\ttrain-merror:0.02199\teval-merror:0.18950\n",
      "[42]\ttrain-merror:0.01885\teval-merror:0.19406\n",
      "[43]\ttrain-merror:0.01828\teval-merror:0.19406\n",
      "[44]\ttrain-merror:0.01799\teval-merror:0.19178\n",
      "[45]\ttrain-merror:0.01628\teval-merror:0.18950\n",
      "[46]\ttrain-merror:0.01656\teval-merror:0.18950\n",
      "[47]\ttrain-merror:0.01428\teval-merror:0.19178\n",
      "[48]\ttrain-merror:0.01314\teval-merror:0.19406\n",
      "[49]\ttrain-merror:0.01199\teval-merror:0.19406\n",
      "[50]\ttrain-merror:0.01114\teval-merror:0.19406\n",
      "[51]\ttrain-merror:0.01028\teval-merror:0.19178\n",
      "[52]\ttrain-merror:0.00885\teval-merror:0.19635\n",
      "[53]\ttrain-merror:0.00885\teval-merror:0.19635\n",
      "[54]\ttrain-merror:0.00857\teval-merror:0.19635\n",
      "[55]\ttrain-merror:0.00771\teval-merror:0.19178\n",
      "[56]\ttrain-merror:0.00685\teval-merror:0.19178\n",
      "[57]\ttrain-merror:0.00657\teval-merror:0.19178\n",
      "[58]\ttrain-merror:0.00514\teval-merror:0.19178\n",
      "[59]\ttrain-merror:0.00428\teval-merror:0.19178\n",
      "[60]\ttrain-merror:0.00400\teval-merror:0.19178\n",
      "[61]\ttrain-merror:0.00343\teval-merror:0.19406\n",
      "[62]\ttrain-merror:0.00371\teval-merror:0.18950\n",
      "[63]\ttrain-merror:0.00314\teval-merror:0.19178\n",
      "[64]\ttrain-merror:0.00257\teval-merror:0.18950\n",
      "[65]\ttrain-merror:0.00228\teval-merror:0.18950\n",
      "[66]\ttrain-merror:0.00228\teval-merror:0.18950\n",
      "[67]\ttrain-merror:0.00200\teval-merror:0.18950\n",
      "[68]\ttrain-merror:0.00200\teval-merror:0.18721\n",
      "[69]\ttrain-merror:0.00200\teval-merror:0.18493\n",
      "[70]\ttrain-merror:0.00171\teval-merror:0.18950\n",
      "[71]\ttrain-merror:0.00143\teval-merror:0.18721\n",
      "[72]\ttrain-merror:0.00114\teval-merror:0.18493\n",
      "[73]\ttrain-merror:0.00114\teval-merror:0.18493\n",
      "[74]\ttrain-merror:0.00114\teval-merror:0.18493\n",
      "[75]\ttrain-merror:0.00114\teval-merror:0.18721\n",
      "[76]\ttrain-merror:0.00057\teval-merror:0.18721\n",
      "[77]\ttrain-merror:0.00057\teval-merror:0.18721\n",
      "[78]\ttrain-merror:0.00057\teval-merror:0.18493\n",
      "[79]\ttrain-merror:0.00057\teval-merror:0.18950\n",
      "[80]\ttrain-merror:0.00057\teval-merror:0.19178\n",
      "[81]\ttrain-merror:0.00057\teval-merror:0.18721\n",
      "[82]\ttrain-merror:0.00057\teval-merror:0.18950\n",
      "[83]\ttrain-merror:0.00057\teval-merror:0.19178\n",
      "[84]\ttrain-merror:0.00057\teval-merror:0.18721\n",
      "[85]\ttrain-merror:0.00057\teval-merror:0.18950\n",
      "[86]\ttrain-merror:0.00057\teval-merror:0.18493\n",
      "[87]\ttrain-merror:0.00057\teval-merror:0.18721\n",
      "[88]\ttrain-merror:0.00057\teval-merror:0.18721\n",
      "[89]\ttrain-merror:0.00057\teval-merror:0.18721\n",
      "[90]\ttrain-merror:0.00057\teval-merror:0.18493\n",
      "[91]\ttrain-merror:0.00029\teval-merror:0.18493\n",
      "[92]\ttrain-merror:0.00029\teval-merror:0.18493\n",
      "[93]\ttrain-merror:0.00029\teval-merror:0.18265\n",
      "[94]\ttrain-merror:0.00029\teval-merror:0.18493\n",
      "[95]\ttrain-merror:0.00029\teval-merror:0.18037\n",
      "[96]\ttrain-merror:0.00029\teval-merror:0.18265\n",
      "[97]\ttrain-merror:0.00029\teval-merror:0.18265\n",
      "[98]\ttrain-merror:0.00029\teval-merror:0.18265\n",
      "[99]\ttrain-merror:0.00029\teval-merror:0.18265\n",
      "CPU times: total: 14.3 s\n",
      "Wall time: 1.22 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# train the models\n",
    "# add the best parameters to the model\n",
    "#best_params = grid_search.best_params_.copy()\n",
    "best_params['objective'] = 'multi:softmax'\n",
    "best_params['eval_metric'] = 'merror'\n",
    "best_params['num_class'] = num_classes\n",
    "\n",
    "num_round = 100\n",
    "\n",
    "# Train the model and get the training history\n",
    "evals_result = {}\n",
    "model = xgb.train(best_params, dtrain, num_round, evals=[(dtrain, 'train'), (dvalid, 'eval')], evals_result=evals_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47280 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get the training loss and validation loss\n",
    "train_loss = evals_result['train']['merror']\n",
    "valid_loss = evals_result['eval']['merror']\n",
    "\n",
    "# Calculate the training accuracy and validation accuracy\n",
    "train_accuracy = [1 - x for x in train_loss]\n",
    "valid_accuracy = [1 - x for x in valid_loss]\n",
    "\n",
    "# Create a new figure\n",
    "fig = plt.figure(figsize=(10, 5))\n",
    "\n",
    "# Plot the training accuracy and validation accuracy\n",
    "plt.plot(train_accuracy, label='Train')\n",
    "plt.plot(valid_accuracy, label='Validation')\n",
    "plt.xlabel('Iteration')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.title('Training Accuracy vs Validation Accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8310502283105022\n"
     ]
    }
   ],
   "source": [
    "# Get the accuracy of the model\n",
    "preds = model.predict(dtest)\n",
    "correct = 0\n",
    "for i in range(len(test_y)):\n",
    "    if preds[i] == test_y[i]:\n",
    "        correct += 1\n",
    "accuracy = correct / len(test_y)\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate Model Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31044 characters]",
      "text/plain": [
       "<Figure size 800x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get the confusion matrix\n",
    "cm = confusion_matrix(test_y, preds)\n",
    "\n",
    "# Create a new figure\n",
    "plt.figure(figsize=(8, 6))\n",
    "\n",
    "labels = ['GSVT', 'AFIB', 'SR', 'SB']\n",
    "# Plot the confusion matrix\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=labels, yticklabels=labels)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('Actual')\n",
    "plt.title('Confusion Matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:title={'center':'Feature importance'}, xlabel='F score', ylabel='Features'>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 46712 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the feature importance\n",
    "xgb.plot_importance(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24804 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot recall and precision\n",
    "# Calculate the recall and precision\n",
    "recall = cm.diagonal() / cm.sum(axis=1)\n",
    "precision = cm.diagonal() / cm.sum(axis=0)\n",
    "\n",
    "# plot in a bar chart\n",
    "fig, ax = plt.subplots(1, 2, figsize=(12, 6))\n",
    "ax[0].bar(range(num_classes), recall)\n",
    "ax[0].set_xticks(range(num_classes))\n",
    "ax[0].set_xticklabels(['GSVT', 'AFIB', 'SR', 'SB'])\n",
    "ax[0].set_xlabel('Class')\n",
    "ax[0].set_ylabel('Recall')\n",
    "ax[0].set_title('Recall')\n",
    "\n",
    "ax[1].bar(range(num_classes), precision)\n",
    "ax[1].set_xticks(range(num_classes))\n",
    "ax[1].set_xticklabels(['GSVT', 'AFIB', 'SR', 'SB'])\n",
    "ax[1].set_xlabel('Class')\n",
    "ax[1].set_ylabel('Precision')\n",
    "ax[1].set_title('Precision')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.8157211953487169\n"
     ]
    }
   ],
   "source": [
    "# Calculate F1 Score for multiclass classification\n",
    "f1 = f1_score(test_y, preds, average='macro')\n",
    "\n",
    "print('F1 Score:', f1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
