{ "cells": [ { "cell_type": "markdown", "id": "005594a6-af0e-4512-897b-f0e5971888a5", "metadata": {}, "source": [ "### Finding similar artists using content-based approach\n", "This notebook currently contains best approach for TF-IDF matrix, decreasing features and calculating/storing\n", "similarites" ] }, { "cell_type": "markdown", "id": "34547dc7-d7f4-4bea-9018-c616e77d5b13", "metadata": {}, "source": [ "To see other approaches and experiments, look at \"Similar Orchard artists algos_multi\" notebook." ] }, { "cell_type": "markdown", "id": "06cf4e70-7641-418e-891f-8f2c47fbe0a7", "metadata": {}, "source": [ "For similarity calculation ANN (approximate nearest neighbor) search algorithm is used. \n", "In this notebook Annoy is implmented although at the end there are also some tests for Faiss and Nmslib algorithms." ] }, { "cell_type": "markdown", "id": "d281a21e-8154-4855-88df-58223b3ebdcc", "metadata": {}, "source": [ "Prepare environment" ] }, { "cell_type": "code", "execution_count": 1, "id": "be69e286-80ad-4dbe-8616-6bc9ed91917a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO: Pandarallel will run on 10 workers.\n", "INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package stopwords to\n", "[nltk_data] /Users/rbomberg/nltk_data...\n", "[nltk_data] Package stopwords is already up-to-date!\n", "[nltk_data] Downloading package punkt to /Users/rbomberg/nltk_data...\n", "[nltk_data] Package punkt is already up-to-date!\n", "[nltk_data] Downloading package omw-1.4 to\n", "[nltk_data] /Users/rbomberg/nltk_data...\n", "[nltk_data] Package omw-1.4 is already up-to-date!\n" ] } ], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", "\n", "\n", "from snowflake.connector.pandas_tools import write_pandas\n", "import numpy as np\n", "from numpy import save, load\n", "import pandas as pd\n", "import datetime as dt\n", "import time\n", "\n", "from pandarallel import pandarallel\n", "pandarallel.initialize(progress_bar=False)\n", "\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "from sklearn.metrics.pairwise import linear_kernel\n", "\n", "from sklearn.neighbors import NearestNeighbors\n", "from sklearn.decomposition import NMF\n", "\n", "from sklearn.decomposition import TruncatedSVD\n", "\n", "import os\n", "import gc\n", "from annoy import AnnoyIndex\n", "import random\n", "\n", "\n", "from matplotlib import pyplot as plt\n", "from IPython.display import HTML\n", "\n", "# enable multiple outputs from single cell\n", "from IPython.core.interactiveshell import InteractiveShell\n", "InteractiveShell.ast_node_interactivity = \"all\"\n", "\n", "%matplotlib inline\n", "plt.style.use('bmh')\n", "\n", "# set display options\n", "pd.set_option('display.max_columns', 20) # default 20\n", "pd.set_option('display.max_colwidth', 150) # default 50\n", "\n", "from utils.functions import *\n", "from utils.sim_functions import *\n", "\n", "# initiate connection\n", "from utils.snowflake_connection import * # check the content of this file to match your profile\n", "ctx, cur = snowflake_key_pair_connect()" ] }, { "cell_type": "markdown", "id": "07336a02-01e1-4ad5-bed5-3560f0a24dda", "metadata": {}, "source": [ "### Prepate dataset for finding Spotify's most similar artists" ] }, { "cell_type": "code", "execution_count": 2, "id": "d97b1da6-578b-4cf1-9892-f1ff23606d6f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading data from parquet file\n", "\n", "RangeIndex: 42706943 entries, 0 to 42706942\n", "Columns: 4 entries, MAIN_ARTIST to FOLLOWERS_LATEST\n", "dtypes: float64(1), object(3)\n", "memory usage: 1.3+ GB\n" ] }, { "data": { "text/plain": [ "(42706943, 4)" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
MAIN_ARTISTARTIST_NAMERELATED_ARTIST_IDFOLLOWERS_LATEST
04t1oCq92CPrgf6Hc9SyN4MEatbananas1EWZHmIFYaKsOWoNSClIyn3.0
\n", "
" ], "text/plain": [ " MAIN_ARTIST ARTIST_NAME RELATED_ARTIST_ID \\\n", "0 4t1oCq92CPrgf6Hc9SyN4M Eatbananas 1EWZHmIFYaKsOWoNSClIyn \n", "\n", " FOLLOWERS_LATEST \n", "0 3.0 " ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "refresh_related_artists = False\n", "# rerun query from db\n", "if refresh_related_artists:\n", " start = dt.datetime.now()\n", "\n", " # query table created with Notebook \"Similar Orchard artists.ipynb\"\n", " sql = \"\"\"select MAIN_ARTIST, ARTIST_NAME, RELATED_ARTIST_ID, to_number(FOLLOWERS_LATEST) as FOLLOWERS_LATEST\n", " from DEV_ENGINEERING.RBOMBERG_DBT.RELATED_ARTISTS\n", " \"\"\"\n", " cur.execute(sql)\n", " related_artists_df = cur.fetch_pandas_all()\n", "\n", " related_artists_df['FOLLOWERS_LATEST'].fillna(0, inplace=True)\n", "\n", " end = dt.datetime.now()\n", " print(f\"Processing took {(end - start).total_seconds() } seconds.\")\n", " \n", " # prepare data for Streamlit\n", " related_artists_df.to_parquet('pickles/related_artists_df.parquet', compression='brotli')\n", "else:\n", " print('Loading data from parquet file')\n", " related_artists_df = pd.read_parquet('pickles/related_artists_df.parquet')\n", " \n", "related_artists_df.info(verbose=False, memory_usage=True)\n", "related_artists_df.shape\n", "related_artists_df.head(1)" ] }, { "cell_type": "code", "execution_count": null, "id": "606e8f1e-c8f5-4d05-8f22-5b43fc98c5b4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "428" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "drop_related_artists_df = False\n", "if drop_related_artists_df:\n", " del related_artists_df\n", " gc.collect()\n", " related_artists_df=pd.DataFrame()" ] }, { "cell_type": "markdown", "id": "b9bd90d8-7314-4d49-8a58-eb5bf359cc61", "metadata": {}, "source": [ "### Load main dataset" ] }, { "cell_type": "code", "execution_count": 9, "id": "d56864d8-830e-4521-a35f-50944d06aa10", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading data from parquet file\n" ] }, { "data": { "text/plain": [ "(8346735, 9)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 8346735 entries, 0 to 8346734\n", "Columns: 9 entries, SPOTIFY_ARTIST_ID to C_BAND\n", "dtypes: int32(1), int8(1), object(7)\n", "memory usage: 485.6+ MB\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SPOTIFY_ARTIST_IDC_FAN_COUNTRY_CODECM_ARTISTC_POPULARITYC_ARTIST_NAMEC_GENRESC_PRONOUNC_GENDERC_BAND
06YQG40LjmFDpqTZUTeMZ9TUS|MX50037971Los Hermanos Perezlatin
\n", "
" ], "text/plain": [ " SPOTIFY_ARTIST_ID C_FAN_COUNTRY_CODE CM_ARTIST C_POPULARITY \\\n", "0 6YQG40LjmFDpqTZUTeMZ9T US|MX 5003797 1 \n", "\n", " C_ARTIST_NAME C_GENRES C_PRONOUN C_GENDER C_BAND \n", "0 Los Hermanos Perez latin " ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "refresh_main_dataframe = False\n", "# rerun query from db\n", "if refresh_main_dataframe:\n", " start = dt.datetime.now()\n", "\n", " # query table created with Notebook \"Similar Orchard artists.ipynb\"\n", " sql = \"\"\"select * from DEV_ENGINEERING.RBOMBERG_DBT.CHARTMERTIC_INPUT_FOR_REC\n", " \"\"\"\n", " cur.execute(sql)\n", " algo_clean_input_df = cur.fetch_pandas_all()\n", " \n", " # Replace unknown values with '' in order to avoid using missing values in similarity calculation\n", " replace_unknowns = True\n", " if replace_unknowns:\n", " algo_clean_input_df.loc[algo_clean_input_df['C_PRONOUN'] == 'other_pronoun', 'C_PRONOUN'] = ''\n", " algo_clean_input_df.loc[algo_clean_input_df['C_GENDER'] == 'gender_unknown', 'C_GENDER'] = ''\n", " algo_clean_input_df.loc[algo_clean_input_df['C_BAND'] == '-1', 'C_BAND'] = ''\n", "\n", " end = dt.datetime.now()\n", " print(f\"Processing took {(end - start).total_seconds() } seconds.\")\n", " \n", " # prepare data for Streamlit\n", " algo_clean_input_df[['SPOTIFY_ARTIST_ID', 'C_FAN_COUNTRY_CODE', 'CM_ARTIST', 'C_POPULARITY',\n", " 'C_ARTIST_NAME', 'C_GENRES', 'C_PRONOUN', 'C_GENDER', 'C_BAND']]\\\n", " .to_parquet('pickles/algo_input_df.parquet', compression='brotli')\n", "else:\n", " print('Loading data from parquet file')\n", " algo_clean_input_df = pd.read_parquet('pickles/algo_input_df.parquet')\n", " \n", "algo_clean_input_df.shape\n", "algo_clean_input_df.info(verbose=False, memory_usage=True)\n", "algo_clean_input_df.head(1)" ] }, { "cell_type": "markdown", "id": "bdcca6a3-6bd2-4dcb-a039-c3bd5bf5b3f7", "metadata": {}, "source": [ "Prepare TF-IDF Vector" ] }, { "cell_type": "code", "execution_count": 10, "id": "05497f93-a50a-481a-9674-a37acc496a77", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Excluding rows where country codes are missing.\n", "After cleaning- nr of rows still missing country code 0.\n", "Left with 5371054 artists for modelling.\n", "Processing took 31.276794 seconds.\n" ] }, { "data": { "text/plain": [ "0 latin US MX \n", "1 pop CA \n", "2 pop christian rbsoul US 0 male \n", "3 arabic IN CZ \n", "4 rock metal US CH MX \n", "Name: COMBINED, dtype: object" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/plain": [ "(5371054, 3197)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Matrix memory usage: 133.63684 megabytes.\n", "Matrix dype: \n", "Nr of TF IDF features: 3197.\n", "First 10 TF IDF features: ['0' '1' '150' '21st' '420' '432hz' '48g' '5th' '6' '8bit'].\n" ] } ], "source": [ "# what columns to use for building TF-IDF\n", "approaches = {\n", " 'a': ['C_GENRES'],\n", " 'b': ['C_GENRES', 'C_FAN_COUNTRY_CODE'],\n", " 'c': ['C_GENRES', 'C_FAN_COUNTRY_CODE', 'C_BAND', 'C_PRONOUN', 'C_GENDER'],\n", " 'd': ['C_FAN_COUNTRY_CODE'],\n", " }\n", "\n", "selection = 'c'\n", "\n", "start = dt.datetime.now()\n", "\n", "# if using approach A then we can keep all data because other approaches require data with country coides\n", "if selection in ['b', 'c', 'd']:\n", " print('Excluding rows where country codes are missing.')\n", " algo_input_df = algo_clean_input_df[~algo_clean_input_df.C_FAN_COUNTRY_CODE.isna()].copy()\n", " algo_input_df.reset_index(inplace=True)\n", " print(f'After cleaning- nr of rows still missing country code {algo_input_df.C_FAN_COUNTRY_CODE.isna().sum()}.')\n", " print(f'Left with {algo_input_df.shape[0]} artists for modelling.')\n", " \n", "# exclude records without proper genre\n", "if selection in ['a']:\n", " print('Excluding rows where genres are not known.')\n", " algo_input_df = algo_clean_input_df[~(algo_clean_input_df['C_GENRES']=='others')].copy()\n", " algo_input_df.reset_index(inplace=True)\n", " print(f'Left with {algo_input_df.shape[0]} artists for modelling.')\n", " \n", "\n", "algo_input_df['C_GENRES'].fillna('', inplace=True)\n", "algo_input_df['C_GENRES'] = algo_input_df['C_GENRES'].parallel_apply(replace_straight_with_space)\n", "algo_input_df['C_FAN_COUNTRY_CODE'] = algo_input_df['C_FAN_COUNTRY_CODE'].parallel_apply(replace_straight_with_space)\n", "\n", "algo_input_df['COMBINED'] = algo_input_df[approaches[selection]].parallel_apply(\n", " lambda x: ' '.join(x.dropna()),\n", " axis=1\n", ")\n", "\n", "end = dt.datetime.now()\n", "print(f\"Processing took {(end - start).total_seconds() } seconds.\")\n", "\n", "tfidf = TfidfVectorizer(analyzer='word',\n", " token_pattern=r'\\w{1,}',\n", " ngram_range=(1, 1), # ngram_range=(1, 3),\n", " stop_words = 'english',\n", " min_df=2) # we don't need features that are present only in single artist\n", "\n", "algo_input_df['COMBINED'].head(5)\n", "\n", "# Fitting the TF-IDF on the selected data\n", "tfidf_matrix = tfidf.fit_transform(algo_input_df['COMBINED'])\n", "\n", "\n", "tfidf_matrix.shape\n", "print(f\"Matrix memory usage: {tfidf_matrix.data.nbytes / 1000000} megabytes.\")\n", "print(f\"Matrix dype: {type(tfidf_matrix)}\")\n", "print(f\"Nr of TF IDF features: {len(tfidf.get_feature_names_out())}.\")\n", "print(f\"First 10 TF IDF features: {tfidf.get_feature_names_out()[0:10]}.\")" ] }, { "cell_type": "markdown", "id": "51cc83f5-5e9e-4b3b-a876-1c676eaee3bb", "metadata": {}, "source": [ "Find optimal SVD value
\n", "We are happy when explained variance is around 95%
\n", "It seems that using only genres requires more components(300) than approaches using more different data." ] }, { "cell_type": "code", "execution_count": 6, "id": "bdb8181a-7ac4-4f71-803c-11b6dcf7fa24", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TruncatedSVD(n_components=10)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of components = 10 and explained variance = 0.3421174832300413\n", "This step took 27.141128 seconds.\n" ] }, { "data": { "text/plain": [ "TruncatedSVD(n_components=50)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of components = 50 and explained variance = 0.7727240063092792\n", "This step took 90.964829 seconds.\n" ] }, { "data": { "text/plain": [ "TruncatedSVD(n_components=100)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of components = 100 and explained variance = 0.8987018216559891\n", "This step took 186.480845 seconds.\n" ] }, { "data": { "text/plain": [ "TruncatedSVD(n_components=150)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of components = 150 and explained variance = 0.9330100819968238\n", "This step took 302.023768 seconds.\n" ] }, { "data": { "text/plain": [ "TruncatedSVD(n_components=200)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of components = 200 and explained variance = 0.9471722302276621\n", "This step took 1164.740357 seconds.\n" ] }, { "data": { "text/plain": [ "TruncatedSVD(n_components=250)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of components = 250 and explained variance = 0.9554275630679595\n", "This step took 901.259969 seconds.\n" ] }, { "data": { "text/plain": [ "TruncatedSVD(n_components=300)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of components = 300 and explained variance = 0.9606864421704278\n", "This step took 1643.581402 seconds.\n" ] } ], "source": [ "n_comp = [10,50,100,150, 200, 250, 300] # list containing different values of components\n", "explained = [] # explained variance ratio for each component of Truncated SVD\n", "for x in n_comp:\n", " start = dt.datetime.now()\n", " svd = TruncatedSVD(n_components=x)\n", " svd.fit(tfidf_matrix)\n", " explained.append(svd.explained_variance_ratio_.sum())\n", " print(\"Number of components = %r and explained variance = %r\"%(x,svd.explained_variance_ratio_.sum()))\n", " end = dt.datetime.now()\n", " print(f\"This step took {(end - start).total_seconds() } seconds.\")\n", " \n", " if svd.explained_variance_ratio_.sum() >= 0.95:\n", " print(\"Found optimal nr of compoents.\")\n", " break" ] }, { "cell_type": "markdown", "id": "7b2b74ec-8af7-4699-8d29-a2c956547556", "metadata": {}, "source": [ "Fill dimensions required for each approach" ] }, { "cell_type": "code", "execution_count": 3, "id": "752f89e2-4676-4abf-8495-e60fc75443c5", "metadata": {}, "outputs": [], "source": [ "reduced_dimensions = {\n", " 'a': 300,\n", " 'b': 200,\n", " 'c': 250, # previously 200\n", " 'd': 200,\n", " }" ] }, { "cell_type": "markdown", "id": "99665acb-4e7d-4fb8-b90b-e5d01adbcc22", "metadata": {}, "source": [ "Perform dimensionality reduction and save matrix for faster testing in the future" ] }, { "cell_type": "code", "execution_count": 4, "id": "36150351-9f83-406c-ad01-a89db30cd665", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading reduced matrix from disk.\n" ] }, { "data": { "text/plain": [ "(5371054, 250)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "New truncated matrix memory usage: 10742.108 megabytes.\n" ] } ], "source": [ "load_truncated_matrix = True\n", "\n", "if load_truncated_matrix:\n", " print('Loading reduced matrix from disk.')\n", " tfidf_matrix_truncated = load('pickles/tfidf_matrix_truncated.npy')\n", "else:\n", " print('Applying dimensionality reduction.')\n", " start = dt.datetime.now()\n", " \n", " if tfidf_matrix.shape[1] > 200:\n", " sparse = False\n", " truncatedSVD = TruncatedSVD(reduced_dimensions.get(selection))\n", " tfidf_matrix_truncated = truncatedSVD.fit_transform(tfidf_matrix)\n", " else:\n", " sparse = True\n", " tfidf_matrix_truncated = tfidf_matrix # we continue using already existing matrix as it's small\n", " \n", " \n", " # save to npy file\n", " save('pickles/tfidf_matrix_truncated.npy', tfidf_matrix_truncated)\n", " \n", " end = dt.datetime.now()\n", " print(f\"Processing took {(end - start).total_seconds() } seconds.\")\n", "\n", "tfidf_matrix_truncated.shape\n", "print(f\"New truncated matrix memory usage: {tfidf_matrix_truncated.data.nbytes / 1000000} megabytes.\")" ] }, { "cell_type": "markdown", "id": "a255e1e9-2587-4f0f-8360-fcfbe46a4a10", "metadata": {}, "source": [ "Build Annoy index
\n", "General ANN reading: https://towardsdatascience.com/comprehensive-guide-to-approximate-nearest-neighbors-algorithms-8b94f057d6b6" ] }, { "cell_type": "code", "execution_count": 7, "id": "6e68879a-f7e6-42b3-89c8-5e3d0079c811", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "------ Starting annoy index building process for approach: c -------\n", "Nr of trees: 75.\n", "Adding items to index took 51.56737 seconds.\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Building took 740.001062 seconds.\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ "Saving took -7.5e-05 seconds.\n", "Ann file size: 10959.844632 megabytes.\n" ] } ], "source": [ "trees = [5, 15, 25, 50]\n", "trees = [75]\n", "selection = 'c'\n", "sparse = False\n", "for tri in trees:\n", " print(f\"------ Starting annoy index building process for approach: {selection} -------\")\n", " print(f\"Nr of trees: {tri}.\")\n", " start = dt.datetime.now()\n", " \n", " t = AnnoyIndex(tfidf_matrix_truncated.shape[1], 'angular')\n", "\n", " for i, row in enumerate(tfidf_matrix_truncated):\n", " if sparse:\n", " t.add_item(i, row.todense().tolist()[0])\n", " else:\n", " t.add_item(i, row.tolist())\n", " \n", " end = dt.datetime.now()\n", " print(f\"Adding items to index took {(end - start).total_seconds() } seconds.\")\n", " \n", " start = dt.datetime.now()\n", " t.build(tri)\n", " end = dt.datetime.now()\n", " print(f\"Building took {(end - start).total_seconds() } seconds.\")\n", " \n", " start = dt.datetime.now()\n", " t.save(f'pickles/{selection}_test.ann')\n", " file_size = os.path.getsize(f'pickles/{selection}_test.ann')\n", " print(f\"Saving took {(end - start).total_seconds() } seconds.\")\n", " print(f\"Ann file size: {file_size / 1000000} megabytes.\")" ] }, { "cell_type": "markdown", "id": "e3b2861f-efc7-492d-a210-5eb4dd8d3c6a", "metadata": {}, "source": [ "Load Annoy index for testing similarity" ] }, { "cell_type": "code", "execution_count": 8, "id": "8f8fbae0-9441-4738-a2b2-b078a9da0da3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "u = AnnoyIndex(reduced_dimensions.get(selection), 'angular')\n", "u.load(f'pickles/{selection}_test.ann')" ] }, { "cell_type": "markdown", "id": "55804e27-3644-4532-8bb3-466d0b891d6e", "metadata": {}, "source": [ "Use list of CM ARTIST gathered from Orchard employees for validation or use popular artists" ] }, { "cell_type": "code", "execution_count": 18, "id": "b64e22a1-05f9-429f-8da2-0d116a96af61", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
C_ARTIST_NAMEC_GENRESC_BANDC_PRONOUNC_GENDERC_POPULARITY
1608985Jinjermetal nu metal1multi53
3022732Oasisbeatlesque permanent wave madchester rock pop1multigender_male75
3552341Floating Pointselectronic jazz dance electronica uk bass microhouse0male51
3604580Taylor Swiftcountry pop rock0femalegender_female100
3197745JP Saxecanadian contemporary rb alt z pop0malegender_male67
5273701Olivia Rodrigopop postteen pop0femalegender_female84
\n", "
" ], "text/plain": [ " C_ARTIST_NAME \\\n", "1608985 Jinjer \n", "3022732 Oasis \n", "3552341 Floating Points \n", "3604580 Taylor Swift \n", "3197745 JP Saxe \n", "5273701 Olivia Rodrigo \n", "\n", " C_GENRES C_BAND \\\n", "1608985 metal nu metal 1 \n", "3022732 beatlesque permanent wave madchester rock pop 1 \n", "3552341 electronic jazz dance electronica uk bass microhouse 0 \n", "3604580 country pop rock 0 \n", "3197745 canadian contemporary rb alt z pop 0 \n", "5273701 pop postteen pop 0 \n", "\n", " C_PRONOUN C_GENDER C_POPULARITY \n", "1608985 multi 53 \n", "3022732 multi gender_male 75 \n", "3552341 male 51 \n", "3604580 female gender_female 100 \n", "3197745 male gender_male 67 \n", "5273701 female gender_female 84 " ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "use_orchard_list = True\n", "if use_orchard_list:\n", " orch_list = [486152, 162652, 297120, 210494, 4563, 194905,\n", " 1958, 206979, 3748501, 437323, 180047, 71305, 207804,\n", " 341882, 567966,\n", " 1615300,\n", " 182078, 209169, 81807, 912, 3353966, 141609, 572217,\n", " 2762, 558681, 5381, 210712, 2581, 260477]\n", " algo_input_df[['C_ARTIST_NAME', 'C_GENRES', 'C_BAND', 'C_PRONOUN', 'C_GENDER', 'C_POPULARITY']][algo_input_df['CM_ARTIST'].\\\n", " isin(orch_list)].sample(n = 6) # pick random artists" ] }, { "cell_type": "code", "execution_count": 103, "id": "f8949d2d-afd5-4c58-b0b9-dcc054f1817f", "metadata": {}, "outputs": [], "source": [ "use_popular_artists = False\n", "if use_popular_artists:\n", " algo_input_df[['C_ARTIST_NAME', 'C_GENRES', 'C_BAND', 'C_PRONOUN', 'C_GENDER']][algo_input_df['C_POPULARITY']<10].sample(n = 6) # pick random artists" ] }, { "cell_type": "markdown", "id": "eb8051a5-ba9d-4d40-b8cb-e3cfdb78dc4b", "metadata": {}, "source": [ "Pick a index value from output above and use as artist_index." ] }, { "cell_type": "code", "execution_count": 19, "id": "f5975ddd-f0ff-4d79-8539-cd7764f6fb5d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Artist with index 3604580 and value 0.0002547894837334752.\n", "Artist with index 60846 and value 0.22161349654197693.\n", "Artist with index 114919 and value 0.22161349654197693.\n", "Artist with index 450906 and value 0.22161349654197693.\n", "Artist with index 1515130 and value 0.22161349654197693.\n", "Artist with index 1975328 and value 0.22161349654197693.\n", "Artist with index 2072102 and value 0.22161349654197693.\n", "Artist with index 2183840 and value 0.22161349654197693.\n", "Artist with index 2367074 and value 0.22161349654197693.\n", "Artist with index 2663666 and value 0.22161349654197693.\n", "Artist with index 2823570 and value 0.22161349654197693.\n", "Artist with index 2842697 and value 0.22161349654197693.\n", "Artist with index 3694739 and value 0.22161349654197693.\n", "Artist with index 3716807 and value 0.22161349654197693.\n", "Artist with index 1943965 and value 0.22163176536560059.\n", "Artist with index 1233794 and value 0.2306307554244995.\n", "Artist with index 420517 and value 0.24595072865486145.\n", "Artist with index 1641836 and value 0.2895885407924652.\n", "Artist with index 292132 and value 0.33559709787368774.\n", "Artist with index 1841816 and value 0.3393596112728119.\n", "Artist with index 3411259 and value 0.34278547763824463.\n", "Artist with index 3501619 and value 0.34278547763824463.\n", "Artist with index 4265840 and value 0.34278547763824463.\n", "Artist with index 1277928 and value 0.36089450120925903.\n", "Artist with index 2645679 and value 0.36788374185562134.\n" ] } ], "source": [ "artist_index = 3604580\n", "nr_of_matches = 25\n", "nearest_artists, distances = u.get_nns_by_item(artist_index, nr_of_matches, include_distances=True, search_k=-1)\n", "\n", "for e, i in enumerate(nearest_artists):\n", " print(f\"Artist with index {nearest_artists[e]} and value {distances[e]}.\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "298976de-4c69-45be-aa35-495dd57d5f31", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Next similarity results were calculated using following columns: ['C_GENRES', 'C_FAN_COUNTRY_CODE', 'C_BAND', 'C_PRONOUN', 'C_GENDER']\n" ] }, { "data": { "text/plain": [ "C_ARTIST_NAME Taylor Swift\n", "C_POPULARITY 100\n", "C_GENRES country pop rock\n", "C_FAN_COUNTRY_CODE US AU GB\n", "C_BAND 0\n", "C_PRONOUN female\n", "C_GENDER gender_female\n", "Name: 3604580, dtype: object" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
C_ARTIST_NAMEC_POPULARITYC_GENRESC_FAN_COUNTRY_CODEC_BANDC_PRONOUNC_GENDER
3604580Taylor Swift100country pop rockUS AU GB0femalegender_female
60846Jordyn Stoddard8pop rock countryUS AU GB0female
114919Stacy Gabel0pop rock countryUS GB AU0female
450906Ramona Rose9country pop rockAU US GB0female
1515130Lauren Bonnell1pop rock countryUS GB AU0female
1975328Cassidy Paris2rock pop countryAU US GB0female
2072102Nicki Kris11pop rock countryGB US AU0female
2183840Jessica Lynne Witty1country pop rockUS GB AU0female
2367074Brooke Law23country pop rockUS AU GB0female
2663666Savannah Jaine14country pop rockGB US AU0female
2823570Charlotte Young1country pop rockGB AU DE0female
2842697Melly Sabine0country rock popGB US AU0female
3694739Shellyann7pop country rockGB DE AU0female
3716807Scarlette Fever5country pop rockUS GB AU0female
1943965Mickie James21wrestling country rock popUS AU GB0female
1233794Lynn Anderson46nashville sound country pop rockAU US GB0female
420517Lauren Marcus35show tunes pop rock countryUS AU GB0female
1641836Ruby Rae5rock countryUS GB AU0femalegender_female
292132Kari Kimmel37pop rock country candy popUS AU GB0female
1841816Skeeter Davis47brill building pop nashville sound country pop rockUS AU GB0female
3411259Lia Caton14country pop rockUS AU DE0female
3501619Kelsey Steele0country pop rockUS AU0female
4265840Emily Clair16country pop rockUS AU IE0female
1277928Lexxi Raine12pop rock countryGB CA AU0female
2645679Annabel0rock countryUS GB AU0female
\n", "
" ], "text/plain": [ " C_ARTIST_NAME C_POPULARITY \\\n", "3604580 Taylor Swift 100 \n", "60846 Jordyn Stoddard 8 \n", "114919 Stacy Gabel 0 \n", "450906 Ramona Rose 9 \n", "1515130 Lauren Bonnell 1 \n", "1975328 Cassidy Paris 2 \n", "2072102 Nicki Kris 11 \n", "2183840 Jessica Lynne Witty 1 \n", "2367074 Brooke Law 23 \n", "2663666 Savannah Jaine 14 \n", "2823570 Charlotte Young 1 \n", "2842697 Melly Sabine 0 \n", "3694739 Shellyann 7 \n", "3716807 Scarlette Fever 5 \n", "1943965 Mickie James 21 \n", "1233794 Lynn Anderson 46 \n", "420517 Lauren Marcus 35 \n", "1641836 Ruby Rae 5 \n", "292132 Kari Kimmel 37 \n", "1841816 Skeeter Davis 47 \n", "3411259 Lia Caton 14 \n", "3501619 Kelsey Steele 0 \n", "4265840 Emily Clair 16 \n", "1277928 Lexxi Raine 12 \n", "2645679 Annabel 0 \n", "\n", " C_GENRES \\\n", "3604580 country pop rock \n", "60846 pop rock country \n", "114919 pop rock country \n", "450906 country pop rock \n", "1515130 pop rock country \n", "1975328 rock pop country \n", "2072102 pop rock country \n", "2183840 country pop rock \n", "2367074 country pop rock \n", "2663666 country pop rock \n", "2823570 country pop rock \n", "2842697 country rock pop \n", "3694739 pop country rock \n", "3716807 country pop rock \n", "1943965 wrestling country rock pop \n", "1233794 nashville sound country pop rock \n", "420517 show tunes pop rock country \n", "1641836 rock country \n", "292132 pop rock country candy pop \n", "1841816 brill building pop nashville sound country pop rock \n", "3411259 country pop rock \n", "3501619 country pop rock \n", "4265840 country pop rock \n", "1277928 pop rock country \n", "2645679 rock country \n", "\n", " C_FAN_COUNTRY_CODE C_BAND C_PRONOUN C_GENDER \n", "3604580 US AU GB 0 female gender_female \n", "60846 US AU GB 0 female \n", "114919 US GB AU 0 female \n", "450906 AU US GB 0 female \n", "1515130 US GB AU 0 female \n", "1975328 AU US GB 0 female \n", "2072102 GB US AU 0 female \n", "2183840 US GB AU 0 female \n", "2367074 US AU GB 0 female \n", "2663666 GB US AU 0 female \n", "2823570 GB AU DE 0 female \n", "2842697 GB US AU 0 female \n", "3694739 GB DE AU 0 female \n", "3716807 US GB AU 0 female \n", "1943965 US AU GB 0 female \n", "1233794 AU US GB 0 female \n", "420517 US AU GB 0 female \n", "1641836 US GB AU 0 female gender_female \n", "292132 US AU GB 0 female \n", "1841816 US AU GB 0 female \n", "3411259 US AU DE 0 female \n", "3501619 US AU 0 female \n", "4265840 US AU IE 0 female \n", "1277928 GB CA AU 0 female \n", "2645679 US GB AU 0 female " ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "main_cols = ['C_ARTIST_NAME', 'C_POPULARITY']\n", "\n", "print(f\"Next similarity results were calculated using following columns: {approaches[selection]}\")\n", "algo_input_df.loc[artist_index][main_cols + approaches[selection]]\n", "algo_input_df.loc[nearest_artists][main_cols + approaches[selection]]" ] }, { "cell_type": "markdown", "id": "32949694-674a-482a-bba8-d9b208bfab8c", "metadata": {}, "source": [ "
" ] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:kdnugget_recommender]", "language": "python", "name": "conda-env-kdnugget_recommender-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }