diff --git a/Fast_Simple_Parallel_KNN_Gzip.ipynb b/Fast_Simple_Parallel_KNN_Gzip.ipynb new file mode 100644 index 0000000..97a2c2f --- /dev/null +++ b/Fast_Simple_Parallel_KNN_Gzip.ipynb @@ -0,0 +1,96 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import gzip\n", + "import pickle\n", + "import numpy as np\n", + "from joblib import Parallel, delayed\n", + "from sklearn.neighbors import KNeighborsClassifier" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.7029702970297029\n" + ] + } + ], + "source": [ + "with open(f\"sentiment-dataset-500.pickle\", \"rb\") as f:\n", + " train_x, train_y, test_x, test_y = pickle.load(f)\n", + "# pre-encode the data, thank you Ken Schutte for this trick\n", + "train_x = [s.encode() for s in train_x]\n", + "test_x = [s.encode() for s in test_x]\n", + "\n", + "def compressed_len(s: bytes) -> int:\n", + " return len(gzip.compress(s))\n", + "\n", + "def ncd_cdist_parallel(x1: list[bytes], x2: list[bytes]) -> np.ndarray:\n", + " n, m = len(x1), len(x2)\n", + " # cache the compressed lengths for min max normalization\n", + " len_x1 = Parallel(n_jobs=-1)(delayed(compressed_len)(s)for s in x1) \n", + " len_x2 = Parallel(n_jobs=-1)(delayed(compressed_len)(s)for s in x2)\n", + " # compute the distance matrix instead of a slow lambda function\n", + " def compute_dist(i: int, j: int) -> float:\n", + " return (compressed_len(x1[i] + b' ' + x2[j]) - min(len_x1[i], len_x2[j])) / max(len_x1[i], len_x2[j])\n", + " # compute the distance matrix in parallel\n", + " dist_mat = Parallel(n_jobs=-1)(delayed(compute_dist)(i, j) for i in range(n) for j in range(m))\n", + " # reshape the distance matrix from flattened to desired shape\n", + " return np.reshape(dist_mat, (n, m))\n", + "\n", + "\n", + "# process the data\n", + "train_ncd = ncd_cdist_parallel(train_x, train_x)\n", + "test_ncd = ncd_cdist_parallel(test_x, train_x)\n", + "# KNN\n", + "neigh = KNeighborsClassifier(n_neighbors=7) \n", + "neigh.fit(train_ncd, train_y)\n", + "print(\"Accuracy:\", neigh.score(test_ncd, test_y))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "time: 8.31 s\n", + "vs \n", + "time: 39 s for video non Parallel code \n", + "vs\n", + "time: 16.3 s for video Parallel code" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}