Fast Simple Parallel Gzip NCD Calculation

Fast_Simple_Parallel_KNN_Gzip.ipynb

@@ -0,0 +1,96 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gzip\n",
+    "import pickle\n",
+    "import numpy as np\n",
+    "from joblib import Parallel, delayed\n",
+    "from sklearn.neighbors import KNeighborsClassifier"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy: 0.7029702970297029\n"
+     ]
+    }
+   ],
+   "source": [
+    "with open(f\"sentiment-dataset-500.pickle\", \"rb\") as f:\n",
+    "    train_x, train_y, test_x, test_y = pickle.load(f)\n",
+    "# pre-encode the data, thank you Ken Schutte for this trick\n",
+    "train_x = [s.encode() for s in train_x]\n",
+    "test_x = [s.encode() for s in test_x]\n",
+    "\n",
+    "def compressed_len(s: bytes) -> int:\n",
+    "    return len(gzip.compress(s))\n",
+    "\n",
+    "def ncd_cdist_parallel(x1: list[bytes], x2: list[bytes]) -> np.ndarray:\n",
+    "    n, m = len(x1), len(x2)\n",
+    "    # cache the compressed lengths for min max normalization\n",
+    "    len_x1 = Parallel(n_jobs=-1)(delayed(compressed_len)(s)for s in x1)  \n",
+    "    len_x2 = Parallel(n_jobs=-1)(delayed(compressed_len)(s)for s in x2)\n",
+    "    # compute the distance matrix instead of a slow lambda function\n",
+    "    def compute_dist(i: int, j: int) -> float:\n",
+    "        return (compressed_len(x1[i] + b' ' + x2[j]) - min(len_x1[i], len_x2[j])) / max(len_x1[i], len_x2[j])\n",
+    "    # compute the distance matrix in parallel\n",
+    "    dist_mat = Parallel(n_jobs=-1)(delayed(compute_dist)(i, j) for i in range(n) for j in range(m))\n",
+    "    # reshape the distance matrix from flattened to desired shape\n",
+    "    return np.reshape(dist_mat, (n, m))\n",
+    "\n",
+    "\n",
+    "# process the data\n",
+    "train_ncd = ncd_cdist_parallel(train_x, train_x)\n",
+    "test_ncd = ncd_cdist_parallel(test_x, train_x)\n",
+    "# KNN\n",
+    "neigh = KNeighborsClassifier(n_neighbors=7) \n",
+    "neigh.fit(train_ncd, train_y)\n",
+    "print(\"Accuracy:\", neigh.score(test_ncd, test_y))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "time: 8.31 s\n",
+    "vs \n",
+    "time: 39 s for video non Parallel code \n",
+    "vs\n",
+    "time: 16.3 s  for video Parallel code"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}