{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import sqlite3\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from pandas.plotting import parallel_coordinates\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.metrics import adjusted_rand_score, normalized_mutual_info_score, silhouette_samples, silhouette_score\n",
    "from sklearn.metrics.pairwise import euclidean_distances\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Data from Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connect to the database\n",
    "conn = sqlite3.connect('../features.db')\n",
    "c = conn.cursor()\n",
    "# get training, validation and test data\n",
    "train = pd.read_sql_query(\"SELECT * FROM train\", conn)\n",
    "valid = pd.read_sql_query(\"SELECT * FROM validation\", conn)\n",
    "test = pd.read_sql_query(\"SELECT * FROM test\", conn)\n",
    "# close the connection\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature names: ['age', 'gender', 'artial_rate', 'ventricular_rate', 'qrs_duration', 'qt_length', 'qrs_count', 'q_peak', 'r_axis', 't_axis']\n",
      "Label names: ['GSVT', 'AFIB', 'SR', 'SB']\n"
     ]
    }
   ],
   "source": [
    "feature_names = train.columns[2:]\n",
    "print('Feature names:', list(feature_names))\n",
    "\n",
    "with open('../settings.json', 'r') as f:\n",
    "    settings = json.load(f)\n",
    "label_names = list(settings['labels'].keys())\n",
    "print('Label names:', label_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Data for CLustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x shape: (4378, 10)\n",
      "features: ['id', 'age', 'gender', 'artial_rate', 'ventricular_rate', 'qrs_duration', 'qt_length', 'qrs_count', 'q_peak', 'r_axis', 't_axis']\n",
      "number of classes: 4\n"
     ]
    }
   ],
   "source": [
    "# get the target and features\n",
    "train_y = train['y']\n",
    "train_y = train_y.map({'GSVT': 0, 'AFIB': 1, 'SR': 2, 'SB': 3})\n",
    "train_x = train.drop(columns=['y'])\n",
    "\n",
    "valid_y = valid['y']\n",
    "valid_y = valid_y.map({'GSVT': 0, 'AFIB': 1, 'SR': 2, 'SB': 3})\n",
    "valid_x = valid.drop(columns=['y'])\n",
    "\n",
    "test_y = test['y']\n",
    "test_y = test_y.map({'GSVT': 0, 'AFIB': 1, 'SR': 2, 'SB': 3})\n",
    "test_x = test.drop(columns=['y'])\n",
    "\n",
    "# add train validation and test data wit concat\n",
    "data_x = pd.concat([train_x, valid_x, test_x], axis=0)\n",
    "data_y = pd.concat([train_y, valid_y, test_y], axis=0)\n",
    "\n",
    "# drop id column\n",
    "data_x = data_x.drop(columns=['id'])\n",
    "print('train_x shape:', data_x.shape)\n",
    "\n",
    "\n",
    "# dealing with missing values\n",
    "# Create an imputer object with a mean filling strategy\n",
    "imputer = SimpleImputer(strategy='mean')\n",
    "data_x = imputer.fit_transform(data_x)\n",
    "\n",
    "# Scale Data between 0 and 1\n",
    "scaler = MinMaxScaler()\n",
    "# Fit the scaler to your data and then transform it\n",
    "data_x = scaler.fit_transform(data_x)\n",
    "# convert to Series\n",
    "data_x = pd.DataFrame(data_x)\n",
    "\n",
    "# print column names\n",
    "print('features:', train_x.columns.to_list())\n",
    "num_classes= len(set(valid_y.to_list()))\n",
    "print('number of classes:', num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster Data with K-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 48676 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "kmeans = KMeans(n_clusters=num_classes, random_state=42)\n",
    "clusters = kmeans.fit_predict(data_x)\n",
    "\n",
    "corr_ma = confusion_matrix(data_y, clusters)\n",
    "sns.heatmap(corr_ma, annot=True, fmt=\"d\", cmap='Blues', yticklabels=label_names)\n",
    "plt.xlabel('K-Means Clusters')\n",
    "plt.ylabel('Diagnosis Group Labels')\n",
    "plt.title('Confusion Matrix of Similiarity between KMeans Clusters and Diagnosis Groups')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0: 1464 patients\n",
      "Cluster 1: 1772 patients\n",
      "Cluster 2: 579 patients\n",
      "Cluster 3: 563 patients\n"
     ]
    }
   ],
   "source": [
    "# Initialize a dictionary to count patients in each cluster\n",
    "cluster_patient_count = {i: 0 for i in range(kmeans.n_clusters)}\n",
    "\n",
    "# Iterate over the assigned clusters and increment the count for each cluster\n",
    "for cluster in clusters:\n",
    "    cluster_patient_count[cluster] += 1\n",
    "\n",
    "# Print the number of patients in each cluster\n",
    "for cluster, count in cluster_patient_count.items():\n",
    "    print(f\"Cluster {cluster}: {count} patients\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 236040 characters]",
      "text/plain": [
       "<Figure size 2000x700 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Apply PCA to reduce to 2 dimensions\n",
    "pca = PCA(n_components=2)\n",
    "reduced_data = pca.fit_transform(data_x)\n",
    "\n",
    "# Create a figure with two subplots\n",
    "fig, axs = plt.subplots(1, 2, figsize=(20, 7))\n",
    "\n",
    "# First subplot for patient labels after PCA reduction\n",
    "for i in range(len(set(data_y))):\n",
    "    cluster_data = reduced_data[data_y == i]\n",
    "    axs[0].scatter(cluster_data[:, 0], cluster_data[:, 1], label=f'Label {label_names[i]}', alpha=0.7, edgecolors='w')\n",
    "\n",
    "axs[0].set_title('Patient Original Labels after Dimensionality Reduction (PCA)')\n",
    "axs[0].set_xlabel('Reduced Dimension 1')\n",
    "axs[0].set_ylabel('Reduced Dimension 2')\n",
    "axs[0].legend()\n",
    "\n",
    "# Second subplot for patient clusters after PCA reduction\n",
    "for i in range(kmeans.n_clusters):\n",
    "    cluster_data = reduced_data[clusters == i]\n",
    "    axs[1].scatter(cluster_data[:, 0], cluster_data[:, 1], label=f'Cluster {i}', alpha=0.7, edgecolors='w')\n",
    "\n",
    "axs[1].set_title('Patient K-Means Clusters after Dimensionality Reduction (PCA)')\n",
    "axs[1].set_xlabel('Reduced Dimension 1')\n",
    "axs[1].set_ylabel('Reduced Dimension 2')\n",
    "axs[1].legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 730896 characters]",
      "text/plain": [
       "<Figure size 2000x700 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA(n_components=3)\n",
    "reduced_data = pca.fit_transform(data_x)\n",
    "\n",
    "fig = plt.figure(figsize=(20, 7))\n",
    "\n",
    "# First subplot\n",
    "ax1 = fig.add_subplot(121, projection='3d')\n",
    "for i in range(kmeans.n_clusters):\n",
    "    cluster_data = reduced_data[clusters == i]\n",
    "    ax1.scatter(cluster_data[:, 0], cluster_data[:, 1], cluster_data[:, 2], label=f'Cluster {i}', alpha=0.7, edgecolors='w')\n",
    "ax1.set_title('Patient K-Means Clusters after Dimensionality Reduction (PCA)')\n",
    "ax1.set_xlabel('Reduced Dimension 1')\n",
    "ax1.set_ylabel('Reduced Dimension 2')\n",
    "ax1.set_zlabel('Reduced Dimension 3')\n",
    "ax1.legend()\n",
    "\n",
    "# Second subplot\n",
    "ax2 = fig.add_subplot(122, projection='3d')\n",
    "for i in range(len(set(data_y))):\n",
    "    cluster_data = reduced_data[data_y == i]\n",
    "    ax2.scatter(cluster_data[:, 0], cluster_data[:, 1], cluster_data[:, 2], label=f'Label {label_names[i]}', alpha=0.7, edgecolors='w') \n",
    "ax2.set_title('Patient Original Labels after Dimensionality Reduction (PCA)')\n",
    "ax2.set_xlabel('Reduced Dimension 1')\n",
    "ax2.set_ylabel('Reduced Dimension 2')\n",
    "ax2.set_zlabel('Reduced Dimension 3')\n",
    "ax2.legend()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adjusted Rand Index (ARI): 0.15412707550646423\n",
      "Normalized Mutual Information (NMI): 0.24282003848756695\n",
      "Silhouette Score: 0.46722973644820026\n"
     ]
    }
   ],
   "source": [
    "# Calculate Adjusted Rand Index (ARI)\n",
    "ari = adjusted_rand_score(data_y, clusters)\n",
    "print(f\"Adjusted Rand Index (ARI): {ari}\")\n",
    "\n",
    "# Calculate Normalized Mutual Information (NMI)\n",
    "nmi = normalized_mutual_info_score(data_y, clusters)\n",
    "print(f\"Normalized Mutual Information (NMI): {nmi}\")\n",
    "\n",
    "# Calculate Silhouette Score\n",
    "silhouette_avg = silhouette_score(data_x, clusters)\n",
    "print(f\"Silhouette Score: {silhouette_avg}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The ARI and NMI scores suggest that the clustering algorithm has some effectiveness in mirroring the true label structure, but it's not highly accurate. The moderate scores indicate that while there is some alignment with the true labels, the clustering does not perfectly capture the underlying groupings.\n",
    "\n",
    "- The Silhouette Score indicates that the clustering has identified groups that are somewhat cohesive internally and separated from each other. This suggests that the clustering algorithm has been somewhat successful in identifying meaningful structures within the data, even if those structures don't perfectly align with the true labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 1444656 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 9.38 s\n",
      "Wall time: 10.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Compute the distance matrix based on the Euclidean distance between data points\n",
    "distance_matrix = euclidean_distances(data_x, data_x)\n",
    "\n",
    "# Create the heatmap\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(distance_matrix, cmap='viridis')\n",
    "plt.title('Heatmap of Euclidean Distances Between Data Points (Patient Features)')\n",
    "plt.xlabel('Data Point Index')\n",
    "plt.ylabel('Data Point Index')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 44220 characters]",
      "text/plain": [
       "<Figure size 600x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_clusters = kmeans.n_clusters\n",
    "# Compute the silhouette scores for each sample\n",
    "silhouette_vals = silhouette_samples(data_x, clusters)\n",
    "\n",
    "# Start plotting\n",
    "fig, ax = plt.subplots(figsize=(6, 8))\n",
    "y_lower, y_upper = 0, 0\n",
    "yticks = []\n",
    "\n",
    "for i, cluster in enumerate(np.unique(clusters)):\n",
    "    cluster_silhouette_vals = silhouette_vals[clusters == cluster]\n",
    "    cluster_silhouette_vals.sort()\n",
    "    y_upper += len(cluster_silhouette_vals)\n",
    "    \n",
    "    color = cm.nipy_spectral(float(i) / n_clusters)\n",
    "    # Ensure the y-axis range matches the length of cluster_silhouette_vals\n",
    "    y_range = np.arange(y_lower, y_lower + len(cluster_silhouette_vals))\n",
    "    ax.fill_betweenx(y_range,\n",
    "                     0, cluster_silhouette_vals,\n",
    "                     facecolor=color, edgecolor=color, alpha=0.7)\n",
    "    \n",
    "    # Label the silhouette plots with their cluster numbers at the middle\n",
    "    ax.text(-0.05, y_lower + 0.5 * len(cluster_silhouette_vals), str(cluster))\n",
    "    \n",
    "    # Compute the new y_lower for next plot\n",
    "    y_lower = y_upper + 10  # 10 for the 0 samples\n",
    "\n",
    "# The vertical line for average silhouette score of all the values\n",
    "average_score = silhouette_score(data_x, clusters)\n",
    "ax.axvline(x=average_score, color=\"red\", linestyle=\"--\")\n",
    "\n",
    "ax.set_yticks([])  # Clear the yaxis clusters / ticks\n",
    "ax.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])\n",
    "plt.title('Silhouette Plot')\n",
    "plt.xlabel('Silhouette Coefficient Values')\n",
    "plt.ylabel('Cluster Label')\n",
    "\n",
    "# Add the silhouette score to the plot\n",
    "plt.text(0.02, 0.95, f'Avg Silhouette Score: {average_score:.2f}', transform=ax.transAxes)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 2007048 characters]",
      "text/plain": [
       "<Figure size 2400x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create a figure with two subplots side by side\n",
    "fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(24, 6))\n",
    "\n",
    "# First subplot for clusters\n",
    "plot_data_clusters = pd.DataFrame(data_x.copy())\n",
    "plot_data_clusters['Cluster'] = clusters\n",
    "new_columns_clusters = plot_data_clusters.columns.tolist()\n",
    "new_columns_clusters[0:10] = feature_names\n",
    "plot_data_clusters.columns = new_columns_clusters\n",
    "parallel_coordinates(plot_data_clusters, 'Cluster', colormap='viridis', ax=axes[0])\n",
    "axes[0].set_title('Feature K-Means Cluster of each Data Point (Patient)')\n",
    "axes[0].set_xlabel('Feature')\n",
    "axes[0].set_ylabel('Feature Value')\n",
    "axes[0].tick_params(axis='x', rotation=90)\n",
    "\n",
    "# Second subplot for labels\n",
    "plot_data_labels = pd.DataFrame(data_x.copy())\n",
    "label_data = data_y.reset_index(drop=True)\n",
    "plot_data_labels['Cluster'] = label_data\n",
    "\n",
    "new_columns_labels = plot_data_labels.columns.tolist()\n",
    "new_columns_labels[0:10] = feature_names\n",
    "plot_data_labels.columns = new_columns_labels\n",
    "parallel_coordinates(plot_data_labels, 'Cluster', colormap='viridis', ax=axes[1])\n",
    "axes[1].set_title('Feature Original Label of each Data Point (Patient)')\n",
    "axes[1].set_xlabel('Feature')\n",
    "axes[1].set_ylabel('Feature Value')\n",
    "axes[1].tick_params(axis='x', rotation=90)\n",
    "# set the legend\n",
    "axes[1].legend(loc='upper right', labels=label_names)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
