{ "cells": [ { "cell_type": "markdown", "id": "f011e49c-a6e7-4b25-a77a-7beb97f7ee98", "metadata": {}, "source": [ "### Notebook for understanding FAISS algorithm" ] }, { "cell_type": "markdown", "id": "6f062ef6-db90-420e-9b03-83f778f72b04", "metadata": {}, "source": [ "Some reading: https://www.pinecone.io/learn/faiss-tutorial/
\n", "and https://www.pinecone.io/learn/vector-indexes/
\n", "and https://github.com/facebookresearch/faiss/wiki/.
" ] }, { "cell_type": "markdown", "id": "57044cd1-2ef8-44c5-81db-9200f92dbd07", "metadata": {}, "source": [ "Prepare environment" ] }, { "cell_type": "code", "execution_count": 1, "id": "e165bb74-9c26-452e-9d20-b20e9a894a87", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO: Pandarallel will run on 10 workers.\n", "INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package stopwords to\n", "[nltk_data] /Users/rbomberg/nltk_data...\n", "[nltk_data] Package stopwords is already up-to-date!\n", "[nltk_data] Downloading package punkt to /Users/rbomberg/nltk_data...\n", "[nltk_data] Package punkt is already up-to-date!\n", "[nltk_data] Downloading package omw-1.4 to\n", "[nltk_data] /Users/rbomberg/nltk_data...\n", "[nltk_data] Package omw-1.4 is already up-to-date!\n" ] } ], "source": [ "import pyarrow.parquet as pq\n", "import pyarrow as pa\n", "import pyarrow.compute as compute\n", "import sys\n", "\n", "import numpy as np\n", "from numpy import save, load\n", "import pandas as pd\n", "import datetime as dt\n", "import time\n", "\n", "from pandarallel import pandarallel\n", "pandarallel.initialize(progress_bar=False)\n", "\n", "import faiss\n", "\n", "from utils.functions import *\n", "\n", "# enable multiple outputs from single cell\n", "from IPython.core.interactiveshell import InteractiveShell\n", "InteractiveShell.ast_node_interactivity = \"all\"" ] }, { "cell_type": "code", "execution_count": 18, "id": "b0921a1d-25b4-4f50-8928-923776351dde", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3256894170" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "675200924" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mmap_related = pa.memory_map('pickles/related_artists_df.arrow')\n", "mmap_main = pa.memory_map('pickles/algo_clean_input_df.arrow')\n", "\n", "table_mmap_related = pa.ipc.RecordBatchFileReader(mmap_related).read_all()\n", "table_mmap_main = pa.ipc.RecordBatchFileReader(mmap_main).read_all()\n", "\n", "\n", "sys.getsizeof(table_mmap_related)\n", "sys.getsizeof(table_mmap_main)" ] }, { "cell_type": "code", "execution_count": 19, "id": "4b47bb5c-9e3e-4212-8a65-169f780c20db", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
MAIN_ARTISTARTIST_NAMERELATED_ARTIST_IDFOLLOWERS_LATEST
04eGgeyC5QrkRPGrMQ5M6zISpace Sailors2M9pkvRd3uUVsPfEgF8aWu1085.0
14eGgeyC5QrkRPGrMQ5M6zINØVMBR0s7medNN59dto0vhQf2jZG1055.0
24eGgeyC5QrkRPGrMQ5M6zIElectronic Gems311L9difAs5Rrx0R8XdX1v820.0
34eGgeyC5QrkRPGrMQ5M6zIenøize3xRoESQLajzT98FLABgTdv730.0
44eGgeyC5QrkRPGrMQ5M6zIcount.003udbEWhlHjgFLXu8fwrHQq663.0
54eGgeyC5QrkRPGrMQ5M6zIEvertt379BjnaqYI8tcQ44FbCUiD598.0
64eGgeyC5QrkRPGrMQ5M6zISpace Cassette101Gki4BZfkxf8iRpNqVCI581.0
74eGgeyC5QrkRPGrMQ5M6zIFujii6xhdKMFpwHJpeNC5FG5lsH498.0
84eGgeyC5QrkRPGrMQ5M6zIIntrospektro0tOETtgtfvo2lPhKkxAbrc426.0
94eGgeyC5QrkRPGrMQ5M6zIT-1202lNxTIinP0mn8ozASWow55418.0
104eGgeyC5QrkRPGrMQ5M6zIRUBYIBIS32cjPtmVtm5yqx33APx5KC409.0
114eGgeyC5QrkRPGrMQ5M6zISørcery3cwfN21ccJYkCDh5MHhl9a399.0
124eGgeyC5QrkRPGrMQ5M6zIVokon3BzqOUdrBlgzCR6MBuZBmH385.0
134eGgeyC5QrkRPGrMQ5M6zIKING PALM54GbY8x05TU5lF1wjGAGwq322.0
144eGgeyC5QrkRPGrMQ5M6zIMEGAS1e9L6JvckFG915fCtQoif6319.0
154eGgeyC5QrkRPGrMQ5M6zIStarLink2ngbe1yDZ3mFkKENa0XQrX312.0
164eGgeyC5QrkRPGrMQ5M6zIiZZi6qe5Wec8BxnB2EgrNRtJAm298.0
174eGgeyC5QrkRPGrMQ5M6zIo b i o4K6CBRrkSqdyfeOoCAfFbn191.0
184eGgeyC5QrkRPGrMQ5M6zICyter6xSvbD19SYZJ2T1bhoYuJR172.0
194eGgeyC5QrkRPGrMQ5M6zIMonolism2AuO7oIWRt96rwS6bd526s146.0
204eGgeyC5QrkRPGrMQ5M6zIParadymCoast7wCHZMQeFNQlQkMCzYalGf136.0
214eGgeyC5QrkRPGrMQ5M6zIMondaze.70T6HAAkI79Bjre6wCcwsU136.0
224eGgeyC5QrkRPGrMQ5M6zIAlbatrauss3BxKxWhor6dSCfMwESPJgX131.0
234eGgeyC5QrkRPGrMQ5M6zIAxium Waves46OBbimbSKPAaVyk24Hx1y109.0
244eGgeyC5QrkRPGrMQ5M6zIDeltavolt7JVTsM5lw7BcXMUDkHd67B92.0
254eGgeyC5QrkRPGrMQ5M6zIFeature Phone6T4igk86vci8rW4sDdOUdP86.0
264eGgeyC5QrkRPGrMQ5M6zICenit854oA33WGYnsqe9zJANC717g83.0
274eGgeyC5QrkRPGrMQ5M6zIHyster6v1MECuMEBJwhCj68MVYJ579.0
284eGgeyC5QrkRPGrMQ5M6zIcirrus5cSIdYPvtxanaGyDF0BddA35.0
\n", "
" ], "text/plain": [ " MAIN_ARTIST ARTIST_NAME RELATED_ARTIST_ID \\\n", "0 4eGgeyC5QrkRPGrMQ5M6zI Space Sailors 2M9pkvRd3uUVsPfEgF8aWu \n", "1 4eGgeyC5QrkRPGrMQ5M6zI NØVMBR 0s7medNN59dto0vhQf2jZG \n", "2 4eGgeyC5QrkRPGrMQ5M6zI Electronic Gems 311L9difAs5Rrx0R8XdX1v \n", "3 4eGgeyC5QrkRPGrMQ5M6zI enøize 3xRoESQLajzT98FLABgTdv \n", "4 4eGgeyC5QrkRPGrMQ5M6zI count.00 3udbEWhlHjgFLXu8fwrHQq \n", "5 4eGgeyC5QrkRPGrMQ5M6zI Evertt 379BjnaqYI8tcQ44FbCUiD \n", "6 4eGgeyC5QrkRPGrMQ5M6zI Space Cassette 101Gki4BZfkxf8iRpNqVCI \n", "7 4eGgeyC5QrkRPGrMQ5M6zI Fujii 6xhdKMFpwHJpeNC5FG5lsH \n", "8 4eGgeyC5QrkRPGrMQ5M6zI Introspektro 0tOETtgtfvo2lPhKkxAbrc \n", "9 4eGgeyC5QrkRPGrMQ5M6zI T-120 2lNxTIinP0mn8ozASWow55 \n", "10 4eGgeyC5QrkRPGrMQ5M6zI RUBYIBIS 32cjPtmVtm5yqx33APx5KC \n", "11 4eGgeyC5QrkRPGrMQ5M6zI Sørcery 3cwfN21ccJYkCDh5MHhl9a \n", "12 4eGgeyC5QrkRPGrMQ5M6zI Vokon 3BzqOUdrBlgzCR6MBuZBmH \n", "13 4eGgeyC5QrkRPGrMQ5M6zI KING PALM 54GbY8x05TU5lF1wjGAGwq \n", "14 4eGgeyC5QrkRPGrMQ5M6zI MEGAS 1e9L6JvckFG915fCtQoif6 \n", "15 4eGgeyC5QrkRPGrMQ5M6zI StarLink 2ngbe1yDZ3mFkKENa0XQrX \n", "16 4eGgeyC5QrkRPGrMQ5M6zI iZZi 6qe5Wec8BxnB2EgrNRtJAm \n", "17 4eGgeyC5QrkRPGrMQ5M6zI o b i o 4K6CBRrkSqdyfeOoCAfFbn \n", "18 4eGgeyC5QrkRPGrMQ5M6zI Cyter 6xSvbD19SYZJ2T1bhoYuJR \n", "19 4eGgeyC5QrkRPGrMQ5M6zI Monolism 2AuO7oIWRt96rwS6bd526s \n", "20 4eGgeyC5QrkRPGrMQ5M6zI ParadymCoast 7wCHZMQeFNQlQkMCzYalGf \n", "21 4eGgeyC5QrkRPGrMQ5M6zI Mondaze. 70T6HAAkI79Bjre6wCcwsU \n", "22 4eGgeyC5QrkRPGrMQ5M6zI Albatrauss 3BxKxWhor6dSCfMwESPJgX \n", "23 4eGgeyC5QrkRPGrMQ5M6zI Axium Waves 46OBbimbSKPAaVyk24Hx1y \n", "24 4eGgeyC5QrkRPGrMQ5M6zI Deltavolt 7JVTsM5lw7BcXMUDkHd67B \n", "25 4eGgeyC5QrkRPGrMQ5M6zI Feature Phone 6T4igk86vci8rW4sDdOUdP \n", "26 4eGgeyC5QrkRPGrMQ5M6zI Cenit85 4oA33WGYnsqe9zJANC717g \n", "27 4eGgeyC5QrkRPGrMQ5M6zI Hyster 6v1MECuMEBJwhCj68MVYJ5 \n", "28 4eGgeyC5QrkRPGrMQ5M6zI cirrus 5cSIdYPvtxanaGyDF0BddA \n", "\n", " FOLLOWERS_LATEST \n", "0 1085.0 \n", "1 1055.0 \n", "2 820.0 \n", "3 730.0 \n", "4 663.0 \n", "5 598.0 \n", "6 581.0 \n", "7 498.0 \n", "8 426.0 \n", "9 418.0 \n", "10 409.0 \n", "11 399.0 \n", "12 385.0 \n", "13 322.0 \n", "14 319.0 \n", "15 312.0 \n", "16 298.0 \n", "17 191.0 \n", "18 172.0 \n", "19 146.0 \n", "20 136.0 \n", "21 136.0 \n", "22 131.0 \n", "23 109.0 \n", "24 92.0 \n", "25 86.0 \n", "26 83.0 \n", "27 79.0 \n", "28 35.0 " ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a = table_mmap_related.filter(compute.equal(table_mmap_related['MAIN_ARTIST'], '4eGgeyC5QrkRPGrMQ5M6zI'))\n", "a.to_pandas()" ] }, { "cell_type": "code", "execution_count": 4, "id": "d7e7c5bf-a13a-43e6-b71e-8877798e8b71", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SPOTIFY_ARTIST_IDC_FAN_COUNTRY_CODECM_ARTISTC_POPULARITYC_ARTIST_NAMEC_GENRESC_PRONOUNC_GENDERC_BAND
054Lqt8wgftvhea92UCEUMPNI|US|MX75203103Sugar Mortemhiphoprap
1411tx2odQnc2P4PuM1ykIzUS|FR|BE83854360Sugar Mittensothers
279XSfxPvUvJ7m0oSoq5ZpDNone42499990Sugar MMFKhiphoprap
33q9KtKE66oCvXeBTHlHUnpIT|US|FR46811530Jah T, Santa Davis, Sugar Minottreggae
40VIeOdcFZz78PHOWyUKnZXUS|GB|CA83499240Sugar Mosesothers
..............................
8143vrdGL94y5b8EpjxqbKYHTR|US|DE42941170Sugar Mary Vartanianothers
823R38PzMIoYiz6tbk613be2JP|US|HK82552840Sugar Mamajpop
833jaSQKGV4QRq8l94gnEyaGUS|CA|NZ93513660Sugar Maple Gospel Choir
845LQQUXlzn0zytdbauckGbYUS|HR|GB53266100Sugar Mikerock|alternative|metal
854Z6qoCbKlPnm8LWOUi1a7dMX|DE|IS36475791Sugar Mistlatin hiphoprap|latin
\n", "

86 rows × 9 columns

\n", "
" ], "text/plain": [ " SPOTIFY_ARTIST_ID C_FAN_COUNTRY_CODE CM_ARTIST C_POPULARITY \\\n", "0 54Lqt8wgftvhea92UCEUMP NI|US|MX 7520310 3 \n", "1 411tx2odQnc2P4PuM1ykIz US|FR|BE 8385436 0 \n", "2 79XSfxPvUvJ7m0oSoq5ZpD None 4249999 0 \n", "3 3q9KtKE66oCvXeBTHlHUnp IT|US|FR 4681153 0 \n", "4 0VIeOdcFZz78PHOWyUKnZX US|GB|CA 8349924 0 \n", ".. ... ... ... ... \n", "81 43vrdGL94y5b8EpjxqbKYH TR|US|DE 4294117 0 \n", "82 3R38PzMIoYiz6tbk613be2 JP|US|HK 8255284 0 \n", "83 3jaSQKGV4QRq8l94gnEyaG US|CA|NZ 9351366 0 \n", "84 5LQQUXlzn0zytdbauckGbY US|HR|GB 5326610 0 \n", "85 4Z6qoCbKlPnm8LWOUi1a7d MX|DE|IS 3647579 1 \n", "\n", " C_ARTIST_NAME C_GENRES C_PRONOUN \\\n", "0 Sugar Mortem hiphoprap \n", "1 Sugar Mittens others \n", "2 Sugar MMFK hiphoprap \n", "3 Jah T, Santa Davis, Sugar Minott reggae \n", "4 Sugar Moses others \n", ".. ... ... ... \n", "81 Sugar Mary Vartanian others \n", "82 Sugar Mama jpop \n", "83 Sugar Maple Gospel Choir \n", "84 Sugar Mike rock|alternative|metal \n", "85 Sugar Mist latin hiphoprap|latin \n", "\n", " C_GENDER C_BAND \n", "0 \n", "1 \n", "2 \n", "3 \n", "4 \n", ".. ... ... \n", "81 \n", "82 \n", "83 \n", "84 \n", "85 \n", "\n", "[86 rows x 9 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "b = table_mmap_main.filter(compute.match_like(table_mmap_main[\"C_ARTIST_NAME\"], \"%Sugar M%\", ignore_case=True))\n", "b.to_pandas()" ] }, { "cell_type": "code", "execution_count": 2, "id": "2a5e72db-90fa-446c-baeb-bff3472ff23c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading reduced matrix from disk.\n" ] }, { "data": { "text/plain": [ "(5371054, 250)" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "dtype('float64')" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Loading main artist data from parquet file\n", "Loading related artist data from parquet file\n" ] } ], "source": [ "print('Loading reduced matrix from disk.')\n", "tfidf_matrix_truncated2 = load('pickles/tfidf_matrix_truncated.npy')\n", "tfidf_matrix_truncated2.shape\n", "tfidf_matrix_truncated2.dtype\n", "\n", "print('Loading main artist data from parquet file')\n", "algo_clean_input_df = pd.read_parquet('pickles/algo_input_df.parquet')\n", "\n", "print('Loading related artist data from parquet file')\n", "related_artists_df = pd.read_parquet('pickles/related_artists_df.parquet')" ] }, { "cell_type": "markdown", "id": "e750d397-033d-45f8-9064-72556ad26e51", "metadata": {}, "source": [ "Save matrix file as raw. Streamlit app requires it to be in a raw format" ] }, { "cell_type": "code", "execution_count": null, "id": "db45f04a-996a-4aaa-b037-26e7ea24a3b3", "metadata": {}, "outputs": [], "source": [ "matrix_to_raw = True\n", "if matrix_to_raw:\n", " tfidf_matrix_truncated2.astype('float64').tofile('pickles/tfidf_matrix_truncated_raw.npy')\n", " \n", " memory_mapped_array = np.memmap('pickles/tfidf_matrix_truncated_raw.npy', dtype='float64', \n", " mode='r+', shape=(5371054,250))\n", " print(\"size mem mapp\", memory_mapped_array.shape)\n", " np.max(memory_mapped_array)\n", " np.min(memory_mapped_array)\n", " memory_mapped_array.size" ] }, { "cell_type": "markdown", "id": "73be5fd3-5655-49d9-8cd1-02733bcd5343", "metadata": {}, "source": [ "Convert from pandas to Arrow if needed" ] }, { "cell_type": "code", "execution_count": 16, "id": "a80cf341-4d0c-4a2c-8469-4f07664747bc", "metadata": {}, "outputs": [], "source": [ "convert_related = True\n", "if convert_related:\n", " table = pa.Table.from_pandas(related_artists_df)\n", " # Write out to file\n", " with pa.OSFile('pickles/related_artists_df.arrow', 'wb') as sink:\n", " with pa.RecordBatchFileWriter(sink, table.schema) as writer:\n", " writer.write_table(table)" ] }, { "cell_type": "code", "execution_count": 3, "id": "81a97aff-7e03-4c35-844b-2ca6feb5b202", "metadata": {}, "outputs": [], "source": [ "approaches = {\n", " 'a': ['C_GENRES'],\n", " 'b': ['C_GENRES', 'C_FAN_COUNTRY_CODE'],\n", " 'c': ['C_GENRES', 'C_FAN_COUNTRY_CODE', 'C_BAND', 'C_PRONOUN', 'C_GENDER'],\n", " 'd': ['C_FAN_COUNTRY_CODE'],\n", " }\n", "\n", "selection = 'c'" ] }, { "cell_type": "code", "execution_count": 4, "id": "1a5cf96d-7f70-4189-8c47-76f4b9e67b30", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Nr of rows before data cleaning: 8346735\n", "Excluding rows where country codes are missing.\n", "After cleaning- nr of rows still missing country code 0.\n", "Left with 5371054 artists for modelling.\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
indexSPOTIFY_ARTIST_IDC_FAN_COUNTRY_CODECM_ARTISTC_POPULARITYC_ARTIST_NAMEC_GENRESC_PRONOUNC_GENDERC_BANDCOMBINED
006YQG40LjmFDpqTZUTeMZ9TUS MX50037971Los Hermanos Perezlatinlatin US MX
1252BZMu6cRBcmeMZAVtD4uwCA49897000Caléropoppop CA
2359cHFsppT290zEGKf0aS8EUS49827301Jay Alliepop christian rbsoulmale0pop christian rbsoul US 0 male
341apBD9QrGTy0Q7BE7A6V6OIN CZ49849250Syed Hasnain Ali Shaharabicarabic IN CZ
450M71g0wCZEMSyql3UPo4FaUS CH MX49859030bastardeyesrock metalrock metal US CH MX
\n", "
" ], "text/plain": [ " index SPOTIFY_ARTIST_ID C_FAN_COUNTRY_CODE CM_ARTIST C_POPULARITY \\\n", "0 0 6YQG40LjmFDpqTZUTeMZ9T US MX 5003797 1 \n", "1 2 52BZMu6cRBcmeMZAVtD4uw CA 4989700 0 \n", "2 3 59cHFsppT290zEGKf0aS8E US 4982730 1 \n", "3 4 1apBD9QrGTy0Q7BE7A6V6O IN CZ 4984925 0 \n", "4 5 0M71g0wCZEMSyql3UPo4Fa US CH MX 4985903 0 \n", "\n", " C_ARTIST_NAME C_GENRES C_PRONOUN C_GENDER C_BAND \\\n", "0 Los Hermanos Perez latin \n", "1 Caléro pop \n", "2 Jay Allie pop christian rbsoul male 0 \n", "3 Syed Hasnain Ali Shah arabic \n", "4 bastardeyes rock metal \n", "\n", " COMBINED \n", "0 latin US MX \n", "1 pop CA \n", "2 pop christian rbsoul US 0 male \n", "3 arabic IN CZ \n", "4 rock metal US CH MX " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "start = dt.datetime.now()\n", "print(\"Nr of rows before data cleaning: \", algo_clean_input_df.shape[0])\n", "replace_unknowns = True\n", "if replace_unknowns:\n", " algo_clean_input_df.loc[algo_clean_input_df['C_PRONOUN'] == 'other_pronoun', 'C_PRONOUN'] = ''\n", " algo_clean_input_df.loc[algo_clean_input_df['C_GENDER'] == 'gender_unknown', 'C_GENDER'] = ''\n", " algo_clean_input_df.loc[algo_clean_input_df['C_BAND'] == '-1', 'C_BAND'] = ''\n", "\n", "# if using approach A then we can keep all data because other approaches require data with country coides\n", "if selection in ['b', 'c', 'd']:\n", " print('Excluding rows where country codes are missing.')\n", " algo_input_df = algo_clean_input_df[~algo_clean_input_df.C_FAN_COUNTRY_CODE.isna()].copy()\n", " algo_input_df.reset_index(inplace=True)\n", " print(f'After cleaning- nr of rows still missing country code {algo_input_df.C_FAN_COUNTRY_CODE.isna().sum()}.')\n", " print(f'Left with {algo_input_df.shape[0]} artists for modelling.')\n", " \n", "# exclude records without proper genre\n", "if selection in ['a']:\n", " print('Excluding rows where genres are not known.')\n", " algo_input_df = algo_clean_input_df[~(algo_clean_input_df['C_GENRES']=='others')].copy()\n", " algo_input_df.reset_index(inplace=True)\n", " print(f'Left with {algo_input_df.shape[0]} artists for modelling.')\n", " \n", "\n", "algo_input_df['C_GENRES'].fillna('', inplace=True)\n", "algo_input_df['C_GENRES'] = algo_input_df['C_GENRES'].parallel_apply(replace_straight_with_space)\n", "algo_input_df['C_FAN_COUNTRY_CODE'] = algo_input_df['C_FAN_COUNTRY_CODE'].parallel_apply(replace_straight_with_space)\n", "\n", "algo_input_df['COMBINED'] = algo_input_df[approaches[selection]].parallel_apply(\n", " lambda x: ' '.join(x.dropna()),\n", " axis=1\n", ")\n", "algo_input_df.head(5)" ] }, { "cell_type": "markdown", "id": "636dfad5-da25-4641-a893-81330b06d4a7", "metadata": {}, "source": [ "Write cleaned file to parquet format if needed" ] }, { "cell_type": "code", "execution_count": 11, "id": "185bcf90-4f4b-4ff7-bd7a-236c7782b854", "metadata": {}, "outputs": [], "source": [ "write_cleaned_to_parquet = True\n", "if write_cleaned_to_parquet:\n", " algo_input_df[['index', 'SPOTIFY_ARTIST_ID', 'C_FAN_COUNTRY_CODE',\n", " 'CM_ARTIST', 'C_POPULARITY', 'C_ARTIST_NAME',\n", " 'C_GENRES', 'C_PRONOUN', 'C_GENDER', 'C_BAND']].to_parquet('pickles/algo_input_df_f.parquet')" ] }, { "cell_type": "markdown", "id": "3be7e744-1bdf-4b23-9328-d6f63bc4b152", "metadata": {}, "source": [ "Convert cleaned file from pandas to Arrow if needed" ] }, { "cell_type": "code", "execution_count": 6, "id": "09577188-df46-4fdc-b618-8da7717c87da", "metadata": {}, "outputs": [], "source": [ "convert_cleaned_arrow = True\n", "if convert_cleaned_arrow:\n", " table = pa.Table.from_pandas(algo_input_df)\n", " # Write out to file\n", " with pa.OSFile('pickles/algo_input_df.arrow', 'wb') as sink:\n", " with pa.RecordBatchFileWriter(sink, table.schema) as writer:\n", " writer.write_table(table)" ] }, { "cell_type": "markdown", "id": "bdd10bb5-3a50-4f9b-96ab-3d42b488fb59", "metadata": {}, "source": [ "TEST DIFFERENT FAISS IMPLEMENTATIONS" ] }, { "cell_type": "markdown", "id": "d288c84d-d08f-4f3f-900a-6a2190e335d0", "metadata": {}, "source": [ "https://github.com/facebookresearch/faiss/wiki
\n", "https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index" ] }, { "cell_type": "code", "execution_count": 6, "id": "7e5c6c98-5eaf-4538-985e-68f213aeb521", "metadata": {}, "outputs": [], "source": [ "# faiss requirement is float32 data type\n", "tfidf_matrix_truncated2 = np.float32(tfidf_matrix_truncated2)" ] }, { "cell_type": "markdown", "id": "6bb26ace-394b-4013-8cb2-8f9b066c7360", "metadata": {}, "source": [ "Simplest approach with flat index
\n", "IndexFlatL2 measures the L2 (or Euclidean) distance between all given points between our query vector, and the vectors loaded into the index. It’s simple, very accurate, but not too fast.
\n", "https://www.pinecone.io/learn/faiss-tutorial/" ] }, { "cell_type": "code", "execution_count": 41, "id": "53dc0470-8986-4f83-a08d-d25ff40eb12d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "5936839" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Building took 5.499074 seconds.\n" ] } ], "source": [ "start = dt.datetime.now()\n", "\n", "# alternatively IndexFlatIP\n", "index = faiss.IndexFlatL2(tfidf_matrix_truncated2.shape[1])\n", "# index = faiss.IndexFlatIP(tfidf_matrix_truncated2.shape[1]) # build the index, d=size of vectors \n", "# here we assume xb contains a n-by-d numpy matrix of type float32\n", "index.add(tfidf_matrix_truncated2) # add vectors to the index\n", "index.ntotal\n", "\n", "faiss.write_index(index, f'pickles/{selection}_faiss')\n", "end = dt.datetime.now()\n", "print(f\"Building took {(end - start).total_seconds() } seconds.\")" ] }, { "cell_type": "markdown", "id": "4aa2a5f7-2808-4d1e-a1cb-9cd373dcba9b", "metadata": {}, "source": [ "This approach for selection a(only genre) produces index file with size 5.38GB on disk." ] }, { "cell_type": "markdown", "id": "a785a421-2c91-43de-bea2-05bead64d2f2", "metadata": {}, "source": [ "Optimize index by applying partitioning / Inverted File Index
\n", "Faiss allows us to add multiple steps that can optimize our search using many different methods. A popular approach is to partition the index into Voronoi cells." ] }, { "cell_type": "markdown", "id": "9cf58378-fa1e-40b2-8ed2-bd22740d6dab", "metadata": {}, "source": [ "If approximate search with IndexIVFFlat returns suboptimal results, we can improve accuracy by increasing the search scope. We do this by increasing the nprobe attribute value — which defines how many nearby cells to search." ] }, { "cell_type": "markdown", "id": "dc458307-d59a-497b-87d2-771e1183fd74", "metadata": {}, "source": [ "A higher nlist means that we must compare our vector to more centroid vectors — but after selecting the nearest centroid’s cells to search, there will be fewer vectors within each cell. So, increase nlist to prioritize search-speed.\n", "\n", "As for nprobe, we find the opposite. Increasing nprobe increases the search scope — thus prioritizing search-quality." ] }, { "cell_type": "code", "execution_count": 7, "id": "f4228ec3-5832-47d1-b6db-6b6705a4e225", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "False" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "True" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "5936839" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Building took 12.170969 seconds.\n" ] } ], "source": [ "start = dt.datetime.now()\n", "\n", "nlist = 50 # to specify how many partitions (Voronoi cells) we’d like our index to have\n", "quantizer = faiss.IndexFlatL2(tfidf_matrix_truncated2.shape[1])\n", "index_f2 = faiss.IndexIVFFlat(quantizer, tfidf_matrix_truncated2.shape[1], nlist)\n", "index_f2.is_trained\n", "\n", "index_f2.train(tfidf_matrix_truncated2)\n", "index_f2.is_trained # check if index is now trained\n", "index_f2.add(tfidf_matrix_truncated2) # add vectors to the index\n", "index_f2.ntotal\n", "faiss.write_index(index_f2, f'pickles/{selection}_quantizer_faiss')\n", "\n", "end = dt.datetime.now()\n", "print(f\"Building took {(end - start).total_seconds() } seconds.\")" ] }, { "cell_type": "markdown", "id": "30657fc2-bf72-47a0-aeb6-040487a0819b", "metadata": {}, "source": [ "This approach for selection a(only genre) produces index file with size 5.42GB on disk" ] }, { "cell_type": "markdown", "id": "be382458-39b8-4bcd-837f-38b62ea2000f", "metadata": {}, "source": [ "Optimization with quantization
\n", "We have one more key optimization to cover. All of our indexes so far have stored our vectors as full (eg Flat) vectors. Now, in very large datasets this can quickly become a problem.\n", "\n", "Fortunately, Faiss comes with the ability to compress our vectors using Product Quantization (PQ)." ] }, { "cell_type": "code", "execution_count": 8, "id": "76f9e599-3563-4c46-94fe-8ca4f7a5feed", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "False" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "True" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Building took 25.444397 seconds.\n" ] } ], "source": [ "m = 10 # number of centroid IDs in final compressed vectors\n", "bits = 8 # number of bits in each centroid\n", "\n", "start = dt.datetime.now()\n", "\n", "quantizer = faiss.IndexFlatL2(tfidf_matrix_truncated2.shape[1]) # we keep the same L2 distance flat index\n", "index_opt = faiss.IndexIVFPQ(quantizer, tfidf_matrix_truncated2.shape[1], nlist, m, bits) \n", "\n", "index_opt.is_trained\n", "\n", "index_opt.train(tfidf_matrix_truncated2)\n", "index_opt.is_trained\n", "\n", "index_opt.add(tfidf_matrix_truncated2)\n", "faiss.write_index(index_opt, f'pickles/{selection}_optimized_faiss')\n", "\n", "end = dt.datetime.now()\n", "print(f\"Building took {(end - start).total_seconds() } seconds.\")" ] }, { "cell_type": "markdown", "id": "70b657ec-a0cb-47ff-8b5d-ea8ad37f69b0", "metadata": {}, "source": [ "This approach for selection a(only genre) produces index file with size 80MB on disk." ] }, { "cell_type": "markdown", "id": "21bf06c8-cfd5-4125-8c38-9b51a6c10fa8", "metadata": {}, "source": [ "Locality Sensitive Hashing
\n", "Our nbits argument refers to the ‘resolution’ of the hashed vectors. A higher value means greater accuracy at the cost of more memory and slower search speeds." ] }, { "cell_type": "code", "execution_count": 9, "id": "48ed755c-8dea-4678-8b4e-e61337b88cd4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Building took 189.611209 seconds.\n" ] } ], "source": [ "nbits = tfidf_matrix_truncated2.shape[1]*4 # resolution of bucketed vectors\n", "# initialize index and add vectors\n", "start = dt.datetime.now()\n", "\n", "index_lsh = faiss.IndexLSH(tfidf_matrix_truncated2.shape[1], nbits)\n", "index_lsh.add(tfidf_matrix_truncated2)\n", "\n", "faiss.write_index(index_lsh, f'pickles/{selection}_lsh_faiss')\n", "\n", "end = dt.datetime.now()\n", "print(f\"Building took {(end - start).total_seconds() } seconds.\")" ] }, { "cell_type": "markdown", "id": "93fdf150-261d-4122-b8bd-5f053d92a56b", "metadata": {}, "source": [ "This approach for selection a(only genre) produces index file with size 700MB on disk" ] }, { "cell_type": "markdown", "id": "27e6ac5c-8753-412b-8dae-727370eb31c4", "metadata": {}, "source": [ "Hierarchical Navigable Small World Graphs
" ] }, { "cell_type": "code", "execution_count": 61, "id": "eeefc8ff-5494-4e0b-aefa-6cf3af65d8d1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Building took 320.841092 seconds.\n" ] } ], "source": [ "# set HNSW index parameters\n", "M = 64 # number of connections each vertex will have\n", "ef_search = 32 # depth of layers explored during search\n", "ef_construction = 8 # depth of layers explored during index construction\n", "\n", "start = dt.datetime.now()\n", "# initialize index\n", "index_hnsw = faiss.IndexHNSWFlat(tfidf_matrix_truncated2.shape[1], M)\n", "# set efConstruction and efSearch parameters\n", "index_hnsw.hnsw.efConstruction = ef_construction\n", "index_hnsw.hnsw.efSearch = ef_search\n", "# add data to index\n", "index_hnsw.add(tfidf_matrix_truncated2)\n", "\n", "faiss.write_index(index_hnsw, f'pickles/{selection}_hnsw_faiss')\n", "end = dt.datetime.now()\n", "print(f\"Building took {(end - start).total_seconds() } seconds.\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "ade61614-8d2b-42e2-a128-f2de34760cf1", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
C_ARTIST_NAMEC_GENRESC_BANDC_PRONOUNC_GENDERC_POPULARITY
3552341Floating Pointselectronic jazz dance electronica uk bass micr...0male51
3197745JP Saxecanadian contemporary rb alt z pop0malegender_male67
2664292The Brian Jonestown Massacrerock shoegaze garage rock bay area indie1multi53
4299034Boards of Canadaelectronic dance ambient scottish electronic t...1multi57
462724Mother's Cakerock austrian stoner rock austrian metal1multi33
1096320Nick Cave & The Bad Seedspop rock blues permanent wave1multi62
\n", "
" ], "text/plain": [ " C_ARTIST_NAME \\\n", "3552341 Floating Points \n", "3197745 JP Saxe \n", "2664292 The Brian Jonestown Massacre \n", "4299034 Boards of Canada \n", "462724 Mother's Cake \n", "1096320 Nick Cave & The Bad Seeds \n", "\n", " C_GENRES C_BAND C_PRONOUN \\\n", "3552341 electronic jazz dance electronica uk bass micr... 0 male \n", "3197745 canadian contemporary rb alt z pop 0 male \n", "2664292 rock shoegaze garage rock bay area indie 1 multi \n", "4299034 electronic dance ambient scottish electronic t... 1 multi \n", "462724 rock austrian stoner rock austrian metal 1 multi \n", "1096320 pop rock blues permanent wave 1 multi \n", "\n", " C_GENDER C_POPULARITY \n", "3552341 51 \n", "3197745 gender_male 67 \n", "2664292 53 \n", "4299034 57 \n", "462724 33 \n", "1096320 62 " ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "use_orchard_list = True\n", "if use_orchard_list:\n", " orch_list = [486152, 162652, 297120, 210494, 4563, 194905,\n", " 1958, 206979, 3748501, 437323, 180047, 71305, 207804,\n", " 341882, 567966,\n", " 1615300,\n", " 182078, 209169, 81807, 912, 3353966, 141609, 572217,\n", " 2762, 558681, 5381, 210712, 2581, 260477]\n", " algo_input_df[['C_ARTIST_NAME', 'C_GENRES', 'C_BAND', 'C_PRONOUN', 'C_GENDER', 'C_POPULARITY']][algo_input_df['CM_ARTIST'].\\\n", " isin(orch_list)].sample(n = 6) # pick random artists" ] }, { "cell_type": "markdown", "id": "f260acdc-01d7-4c37-92f2-c8bf3131fab4", "metadata": {}, "source": [ "Build and compare results from different Faiss implementations
\n", "Use FlatIndex as benchmark" ] }, { "cell_type": "code", "execution_count": 65, "id": "1d3bf844-d394-4373-ad5b-25bd0d39e848", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import gc\n", "del index_hnsw\n", "gc.collect()\n", "index_hnsw=pd.DataFrame()" ] }, { "cell_type": "code", "execution_count": 62, "id": "833bd8c7-fed9-49c4-b47d-bce3b0818232", "metadata": {}, "outputs": [], "source": [ "# dict_flat = {}\n", "# ivf_flat = {}\n", "# opt_flat = {}\n", "# lsh_flat = {}\n", "hnswf_flat = {}" ] }, { "cell_type": "code", "execution_count": 43, "id": "c75beb06-d913-44c7-ab46-8e4c2e6bc574", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "25" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "random_list = algo_input_df[~algo_input_df.C_GENRES.isin([''])].sample(n = 25).index.values.tolist()\n", "len(random_list)" ] }, { "cell_type": "code", "execution_count": 63, "id": "6f8e0a70-893f-4802-87b2-aa88c2f4b446", "metadata": {}, "outputs": [], "source": [ "artists_for_validation = random_list # orch_list\n", "k = 50\n", "index_opt.nprobe = 10\n", "for ar in artists_for_validation:\n", " D, I = index_hnsw.search(tfidf_matrix_truncated2[[ar]], k)\n", " hnswf_flat[ar] = I.tolist()[0]" ] }, { "cell_type": "code", "execution_count": 64, "id": "495afecf-8314-4153-b607-51060a48d8af", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
partitionedcompressedlshhnsw
43388950.020.520.740.00
16265560.100.920.780.00
47149861.001.001.000.10
46622951.001.001.000.00
27739271.001.001.000.00
1316221.000.480.900.88
53361141.001.001.000.00
13710871.001.001.000.30
6212880.940.921.000.02
57224061.000.800.900.00
17284921.001.001.000.00
13928761.000.521.000.08
36545781.001.001.000.00
56942011.001.001.000.00
51307721.001.001.000.38
8357740.820.280.200.00
4134531.001.001.000.00
59367371.001.001.000.00
49224700.980.940.940.00
11750830.100.780.860.02
13755311.001.001.000.00
48319481.001.000.980.00
42077940.940.220.800.00
50861910.500.320.160.00
55500761.001.001.000.00
\n", "
" ], "text/plain": [ " partitioned compressed lsh hnsw\n", "4338895 0.02 0.52 0.74 0.00\n", "1626556 0.10 0.92 0.78 0.00\n", "4714986 1.00 1.00 1.00 0.10\n", "4662295 1.00 1.00 1.00 0.00\n", "2773927 1.00 1.00 1.00 0.00\n", "131622 1.00 0.48 0.90 0.88\n", "5336114 1.00 1.00 1.00 0.00\n", "1371087 1.00 1.00 1.00 0.30\n", "621288 0.94 0.92 1.00 0.02\n", "5722406 1.00 0.80 0.90 0.00\n", "1728492 1.00 1.00 1.00 0.00\n", "1392876 1.00 0.52 1.00 0.08\n", "3654578 1.00 1.00 1.00 0.00\n", "5694201 1.00 1.00 1.00 0.00\n", "5130772 1.00 1.00 1.00 0.38\n", "835774 0.82 0.28 0.20 0.00\n", "413453 1.00 1.00 1.00 0.00\n", "5936737 1.00 1.00 1.00 0.00\n", "4922470 0.98 0.94 0.94 0.00\n", "1175083 0.10 0.78 0.86 0.02\n", "1375531 1.00 1.00 1.00 0.00\n", "4831948 1.00 1.00 0.98 0.00\n", "4207794 0.94 0.22 0.80 0.00\n", "5086191 0.50 0.32 0.16 0.00\n", "5550076 1.00 1.00 1.00 0.00" ] }, "execution_count": 64, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "partitioned 0.8560\n", "compressed 0.8280\n", "lsh 0.8904\n", "hnsw 0.0712\n", "dtype: float64" ] }, "execution_count": 64, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "partitioned 0.313050\n", "compressed 0.265016\n", "lsh 0.227751\n", "hnsw 0.193570\n", "dtype: float64" ] }, "execution_count": 64, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results_dict = {}\n", "for ar in artists_for_validation:\n", " # print(f'artist {ar}')\n", " collected_dict = {}\n", " base_ = dict_flat.get(ar)\n", " len_base = len(base_)\n", " for e, di in enumerate([ivf_flat, opt_flat, lsh_flat, hnswf_flat]):\n", " comp_ = di.get(ar)\n", " common = set(base_).intersection(comp_)\n", " accuracy = len(common) / len_base\n", " collected_dict[e] = accuracy\n", " \n", " results_dict[ar] = collected_dict\n", " \n", "r_df = pd.DataFrame.from_dict(results_dict, orient='index')\n", "r_df.columns= ['partitioned', 'compressed', 'lsh', 'hnsw']\n", "r_df\n", "r_df.mean()\n", "r_df.std()" ] }, { "cell_type": "markdown", "id": "705170c3-0b92-4505-91f2-501a914a08ff", "metadata": {}, "source": [ "Test index" ] }, { "cell_type": "code", "execution_count": null, "id": "5c0b066a-1024-4f9c-9124-d5c097611e20", "metadata": {}, "outputs": [], "source": [ "array([[ 0., 119., 121., 125., 129., 129., 129., 129., 130., 131., 131.,\n", " 133., 133., 135., 135., 135., 135., 136., 136., 136., 137., 137.,\n", " 137., 137., 137.]], dtype=float32)\n", "\n", "array([[4898134, 3271191, 4279863, 3725298, 1515198, 3674856, 123091,\n", " 1756327, 2141655, 3110275, 807282, 4003425, 833096, 2004073,\n", " 2979024, 2717287, 3026522, 1285242, 27343" ] }, { "cell_type": "code", "execution_count": 16, "id": "df9b56e5-7e72-43dc-852f-69890bebf874", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
indexSPOTIFY_ARTIST_IDC_FAN_COUNTRY_CODECM_ARTISTC_POPULARITYC_ARTIST_NAMEC_GENRESC_PRONOUNC_GENDERC_BANDCOMBINED
4441511594449665OrpUcWcHnlcoZsGnZnwmEE FI BR13922223Susanna Aleksandrajazz estonian jazzfemale0jazz estonian jazz EE FI BR 0 female
\n", "
" ], "text/plain": [ " index SPOTIFY_ARTIST_ID C_FAN_COUNTRY_CODE CM_ARTIST \\\n", "4441511 5944496 65OrpUcWcHnlcoZsGnZnwm EE FI BR 1392222 \n", "\n", " C_POPULARITY C_ARTIST_NAME C_GENRES C_PRONOUN \\\n", "4441511 3 Susanna Aleksandra jazz estonian jazz female \n", "\n", " C_GENDER C_BAND COMBINED \n", "4441511 0 jazz estonian jazz EE FI BR 0 female " ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "algo_input_df[algo_input_df['C_ARTIST_NAME'].isin(['Susanna Aleksandra'])]" ] }, { "cell_type": "code", "execution_count": 7, "id": "71cfb03e-0b98-453d-bdd9-fc8794e45f7a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
indexSPOTIFY_ARTIST_IDC_FAN_COUNTRY_CODECM_ARTISTC_POPULARITYC_ARTIST_NAMEC_GENRESC_PRONOUNC_GENDERC_BANDCOMBINED
172611626848516KImCVD70vtIoJWnq6nGn3US MX AU55868190Harry Stylespop instrumentalmalegender_male0pop instrumental US MX AU 0 male gender_male
\n", "
" ], "text/plain": [ " index SPOTIFY_ARTIST_ID C_FAN_COUNTRY_CODE CM_ARTIST \\\n", "1726116 2684851 6KImCVD70vtIoJWnq6nGn3 US MX AU 558681 \n", "\n", " C_POPULARITY C_ARTIST_NAME C_GENRES C_PRONOUN C_GENDER \\\n", "1726116 90 Harry Styles pop instrumental male gender_male \n", "\n", " C_BAND COMBINED \n", "1726116 0 pop instrumental US MX AU 0 male gender_male " ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "algo_input_df[algo_input_df['C_ARTIST_NAME']=='Harry Styles']" ] }, { "cell_type": "code", "execution_count": 106, "id": "8f8eb9fa-54d5-4ec7-93de-5bb20a09c7fd", "metadata": {}, "outputs": [ { "ename": "TypeError", "evalue": "in method 'IndexLSH_search', argument 3 of type 'float const *'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", "Input \u001b[0;32mIn [106]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m#index_opt.nprobe = 10\u001b[39;00m\n\u001b[1;32m 5\u001b[0m k \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m25\u001b[39m \u001b[38;5;66;03m# we want to see K nearest neighbors\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m D, I \u001b[38;5;241m=\u001b[39m \u001b[43mindex_lsh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtfidf_matrix_truncated2\u001b[49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43martist_index\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# actual search\u001b[39;00m\n\u001b[1;32m 8\u001b[0m D\n\u001b[1;32m 9\u001b[0m I\n", "File \u001b[0;32m/opt/homebrew/anaconda3/envs/kdnugget_recommender/lib/python3.8/site-packages/faiss/__init__.py:322\u001b[0m, in \u001b[0;36mhandle_Index..replacement_search\u001b[0;34m(self, x, k, D, I)\u001b[0m\n\u001b[1;32m 319\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 320\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m I\u001b[38;5;241m.\u001b[39mshape \u001b[38;5;241m==\u001b[39m (n, k)\n\u001b[0;32m--> 322\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch_c\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswig_ptr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswig_ptr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mD\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswig_ptr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mI\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 323\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m D, I\n", "File \u001b[0;32m/opt/homebrew/anaconda3/envs/kdnugget_recommender/lib/python3.8/site-packages/faiss/swigfaiss.py:3271\u001b[0m, in \u001b[0;36mIndexLSH.search\u001b[0;34m(self, n, x, k, distances, labels)\u001b[0m\n\u001b[1;32m 3270\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msearch\u001b[39m(\u001b[38;5;28mself\u001b[39m, n, x, k, distances, labels):\n\u001b[0;32m-> 3271\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_swigfaiss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexLSH_search\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdistances\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[0;31mTypeError\u001b[0m: in method 'IndexLSH_search', argument 3 of type 'float const *'" ] } ], "source": [ "artist_index = 1726116\n", "\n", "#index_opt.nprobe = 10\n", "\n", "k = 25 # we want to see K nearest neighbors\n", "D, I = index_lsh.search(tfidf_matrix_truncated2[[artist_index]], k) # actual search\n", "\n", "D\n", "I\n", "faiss_index = I.tolist()[0]" ] }, { "cell_type": "code", "execution_count": 28, "id": "97187007-b0d0-4840-8ad6-bb7c72b78728", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "C_ARTIST_NAME Harry Styles\n", "C_POPULARITY 90\n", "C_GENRES pop instrumental\n", "C_FAN_COUNTRY_CODE US MX AU\n", "C_BAND 0\n", "C_PRONOUN male\n", "C_GENDER gender_male\n", "Name: 1726116, dtype: object" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
C_ARTIST_NAMEC_POPULARITYC_GENRESC_FAN_COUNTRY_CODEC_BANDC_PRONOUNC_GENDER
4471555Jeb Barry and The Pawn Shop Saints0singersongwriter western mass indieUS IN
3397408Paul Jacobsen and the Madison Arm1rock singersongwriter slc indieUS SE DE
2741555Markham Sound0rock singersongwriter slc indieUS DE SE
63156Carson Metzger0rock singersongwriter albuquerque indieUS SE
4100915Mareld0gothenburg indie rock singersongwriterSE US
4213246beißpony0singersongwriter munich indieDE US IT
2152934Cardboard Ox0rhode island indie singersongwriterUS
2140193Song X1singersongwriter dresden indieDE AT US
3676644Ludovan0milan indie singersongwriterIT
4202858Hot Super Hot0singersongwriter edmonton indieUS
138455Johnny Bell0singersongwriter santa fe indieUS
3588841Chunky Planet0cork indie singersongwriterUS DE IE
5306819Jacob Sommerio0singersongwriter charlottesville indieUS IN DE
4914341The Dyr Sister0hull indie singersongwriterDE SE
3532217One Girl Shy0bangalore indie singersongwriterIN US
5146663nattklang0norrbotten indie singersongwriterSE NO
1518308Gebrüder Jakob0singersongwriter graz indieAT
4850840Yarms0singersongwriter rochester ny indieUS
4909374Barty's Path0singersongwriter mumbai indieIN
2215299Sarah Krueger0duluth indie singersongwriterUS DE IT
5012111Bernd Salat0singersongwriter vienna indieAT DE US
1134505Marielle & Co0singersongwriter norrbotten indieSE
1487980Kim Olve Breistrand0singersongwriter trondheim indieNO
292951Inge Chiles0rock singersongwriter springfield mo indieUS
258864Timmy Miano0rock singersongwriter springfield mo indieUS
\n", "
" ], "text/plain": [ " C_ARTIST_NAME C_POPULARITY \\\n", "4471555 Jeb Barry and The Pawn Shop Saints 0 \n", "3397408 Paul Jacobsen and the Madison Arm 1 \n", "2741555 Markham Sound 0 \n", "63156 Carson Metzger 0 \n", "4100915 Mareld 0 \n", "4213246 beißpony 0 \n", "2152934 Cardboard Ox 0 \n", "2140193 Song X 1 \n", "3676644 Ludovan 0 \n", "4202858 Hot Super Hot 0 \n", "138455 Johnny Bell 0 \n", "3588841 Chunky Planet 0 \n", "5306819 Jacob Sommerio 0 \n", "4914341 The Dyr Sister 0 \n", "3532217 One Girl Shy 0 \n", "5146663 nattklang 0 \n", "1518308 Gebrüder Jakob 0 \n", "4850840 Yarms 0 \n", "4909374 Barty's Path 0 \n", "2215299 Sarah Krueger 0 \n", "5012111 Bernd Salat 0 \n", "1134505 Marielle & Co 0 \n", "1487980 Kim Olve Breistrand 0 \n", "292951 Inge Chiles 0 \n", "258864 Timmy Miano 0 \n", "\n", " C_GENRES C_FAN_COUNTRY_CODE C_BAND \\\n", "4471555 singersongwriter western mass indie US IN \n", "3397408 rock singersongwriter slc indie US SE DE \n", "2741555 rock singersongwriter slc indie US DE SE \n", "63156 rock singersongwriter albuquerque indie US SE \n", "4100915 gothenburg indie rock singersongwriter SE US \n", "4213246 singersongwriter munich indie DE US IT \n", "2152934 rhode island indie singersongwriter US \n", "2140193 singersongwriter dresden indie DE AT US \n", "3676644 milan indie singersongwriter IT \n", "4202858 singersongwriter edmonton indie US \n", "138455 singersongwriter santa fe indie US \n", "3588841 cork indie singersongwriter US DE IE \n", "5306819 singersongwriter charlottesville indie US IN DE \n", "4914341 hull indie singersongwriter DE SE \n", "3532217 bangalore indie singersongwriter IN US \n", "5146663 norrbotten indie singersongwriter SE NO \n", "1518308 singersongwriter graz indie AT \n", "4850840 singersongwriter rochester ny indie US \n", "4909374 singersongwriter mumbai indie IN \n", "2215299 duluth indie singersongwriter US DE IT \n", "5012111 singersongwriter vienna indie AT DE US \n", "1134505 singersongwriter norrbotten indie SE \n", "1487980 singersongwriter trondheim indie NO \n", "292951 rock singersongwriter springfield mo indie US \n", "258864 rock singersongwriter springfield mo indie US \n", "\n", " C_PRONOUN C_GENDER \n", "4471555 \n", "3397408 \n", "2741555 \n", "63156 \n", "4100915 \n", "4213246 \n", "2152934 \n", "2140193 \n", "3676644 \n", "4202858 \n", "138455 \n", "3588841 \n", "5306819 \n", "4914341 \n", "3532217 \n", "5146663 \n", "1518308 \n", "4850840 \n", "4909374 \n", "2215299 \n", "5012111 \n", "1134505 \n", "1487980 \n", "292951 \n", "258864 " ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "main_cols = ['C_ARTIST_NAME', 'C_POPULARITY']\n", "algo_input_df.loc[artist_index][main_cols + approaches[selection]]\n", "algo_input_df.loc[faiss_index][main_cols + approaches[selection]]" ] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:kdnugget_recommender]", "language": "python", "name": "conda-env-kdnugget_recommender-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }