{ "cells": [ { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import pandas as pd \n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import math\n", "import datetime\n", "from string import ascii_letters\n", "from dateutil.relativedelta import relativedelta\n", "import math\n", "import pickle as pkl\n", "import swat\n", "import warnings\n", "from tqdm import tqdm\n", "import plotly.graph_objects as go\n", "import plotly.express as px\n", "import psycopg2\n", "import plotly\n", "warnings.filterwarnings('ignore')\n", "pd.set_option('display.max_columns', None)\n", "# sns.set_style('darkgrid')\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "path = '/Users/zhan009/Desktop/anomaly-detection/'\n", "data_1 = pd.read_csv(path+'2017_06_01-2017_12_31.csv', index_col=0)\n", "data_2 = pd.read_csv(path+'2018_01_01-2018_06_30.csv', index_col=0)\n", "data_3 = pd.read_csv(path+'2018_07_01-2018_09_30.csv', index_col=0)\n", "data_4 = pd.read_csv(path+'us_stream_primary_name.csv', index_col=0)\n", "\n", "\n", "all_data = pd.concat([data_1, data_2, data_3, data_4], axis = 0)\n", "stream_data = all_data.sort_values(['primary_artist_name', 'track_isrc', 'min_report_date', 'track_name'])\n", "\n", "\n", "stream_data = stream_data.drop('track_album_artist', axis = 1)\n", "\n", "stream_data['cumsum_streams'] = stream_data.groupby(['primary_artist_name', 'track_name', 'min_report_date', 'track_isrc'])['streams'].cumsum(skipna=True)\n", "stream_data['cumsum_streams'] = np.log10(stream_data['cumsum_streams'])\n", "\n", "cumsum_table = stream_data.drop('streams', axis = 1) ### Table for cumulative sum\n", "pct_table = stream_data.drop('cumsum_streams', axis = 1) ### Table for percentage change" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### Stream Count" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def revert_data(dataset, columns):\n", " dataset = pd.pivot_table(dataset, index=['primary_artist_name', 'track_isrc', 'track_name', 'min_report_date'], columns=['date_diff'], values=columns)\n", " dataset = dataset.reset_index()\n", " return dataset\n", "\n", "####### Cumulative sum table \n", "cumsum_table = revert_data(cumsum_table, 'cumsum_streams')\n", "\n", "###### Percentage change of stream\n", "percent_change = pct_table.copy()\n", "percent_change['streams'] = percent_change['streams']+1\n", "percent_change['pct_change'] = percent_change.groupby(['primary_artist_name', 'track_isrc', 'track_name', 'min_report_date'])['streams'].pct_change()\n", "pct_table = revert_data(percent_change, 'pct_change')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Artist VS All Artists Under Sony " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- Example: Lil Nas X, or Camila Cebello" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- First, get the 99 percentile for all the artist for 90 days. " ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "def get_artist_data(artist_name, dataset):\n", " artist_data = dataset[dataset.primary_artist_name == artist_name]\n", " return artist_data\n", "\n", "def calculate_quantile(artist_name, dataset, quantile_level, exclude = True):\n", " if exclude:\n", " artist_quantile = dataset[dataset.primary_artist_name != artist_name]\n", " else:\n", " artist_quantile = dataset[dataset.primary_artist_name == artist_name]\n", " all_artist_quantile = artist_quantile.quantile(quantile_level, numeric_only=True)\n", " artist_quantile_dict = dict(zip(all_artist_quantile.index, all_artist_quantile.values))\n", "\n", " return all_artist_quantile, artist_quantile_dict\n", "\n", "def remove_duplicate(dataset):\n", " artist_data = pd.DataFrame()\n", " for i in set(dataset.track_isrc):\n", " track_data = dataset[dataset.track_isrc == i]\n", " date = track_data.groupby(['track_name', 'primary_artist_name', 'track_isrc'])['min_report_date'].min().values\n", " track_data = track_data[track_data['min_report_date'] == date[0]]\n", " artist_data = artist_data.append(track_data)\n", " return artist_data\n", "\n", "def quantile_com_all_sony_artist(x, cal_dict):\n", " if x.dtype != np.number:\n", " return x\n", " else:\n", " x = x.where(x>(cal_dict[x.name]))\n", " return x\n", " \n", "def get_result(dataset, quantile_data, label, start_day, quantile):\n", "# plt.figure(figsize=(20, 10))\n", " count = 0 \n", " for i in dataset.track_isrc:\n", " daily = dataset[dataset.track_isrc == i]\n", " streams = daily[range(start_day, 91)]\n", " streams = streams.dropna(axis = 1)\n", " if not streams.empty:\n", " if streams.shape[1] > 5:\n", " count += 1 \n", " print(\"Track name: {0}; {1} days (from Day {2} to Day {3})that surpass the {4}% quantile for all tracks\".format(track_info[i], \\\n", " streams.shape[1], \\\n", " streams.columns[0], \\\n", " streams.columns[-1], \\\n", " quantile*100))\n", "# sns.lineplot(list(streams.columns)[start_day:], streams.values[0][start_day:], label = track_info[i])\n", "# fig.add_trace(go.Scatter(x=list(streams.columns)[start_day:], \n", "# y=streams.values[0][start_day:], \n", "# name=track_info[i],\n", "# line=dict(color='green', width=3),\n", "# mode = 'lines', \n", "# connectgaps=True))\n", " if count < 0:\n", " print(\"There is no track that the streams is surpass the {0}% quantile for all tracks\".format(quantile*100))\n", "# sns.lineplot(range(start_day, 91), quantile_data[start_day:], label = label,lw = 3, color = 'yellow', legend='full')\n", "# fig.add_trace(go.Scatter(x=list(range(start_day, 91)), y=quantile_data[start_day:], \\\n", "# name= '{0}% for all Other Sony Artists'.format(quantile * 100), \\\n", "# line=dict(color='yellow', width=3), line_shape='spline', \\\n", "# mode = 'lines', \\\n", "# connectgaps=True,))\n", "# else:\n", "# print(\"There is no track that the streams is surpass the {0}% quantile for all tracks\".format(quantile*100))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Track name: OMG; 48 days (from Day 43 to Day 90)that surpass the 99.5% quantile for all tracks\n", "Track name: Havana; 71 days (from Day 19 to Day 90)that surpass the 99.5% quantile for all tracks\n", "Track name: Never Be the Same; 11 days (from Day 14 to Day 24)that surpass the 99.5% quantile for all tracks\n", "Track name: Liar; 85 days (from Day 1 to Day 86)that surpass the 99.5% quantile for all tracks\n", "Track name: She Loves Control; 37 days (from Day 53 to Day 90)that surpass the 99.5% quantile for all tracks\n", "Track name: Crying in the Club; 80 days (from Day 11 to Day 90)that surpass the 99.5% quantile for all tracks\n" ] } ], "source": [ "if __name__ == '__main__':\n", " artist_name = 'Camila Cabello'\n", " quantile = 0.995\n", "\n", " artist_data_percentile_old = get_artist_data(artist_name, cumsum_table)\n", " artist_data_percentile_pct_old = get_artist_data(artist_name, pct_table)\n", " artist_data_percentile = remove_duplicate(artist_data_percentile_old)\n", " artist_data_percentile_pct = remove_duplicate(artist_data_percentile_pct_old)\n", "\n", " ###### track info (dictionary{key:value})\n", " track_info = dict(zip(artist_data_percentile.track_isrc, artist_data_percentile.track_name))\n", "\n", " ###### Artist Stream Quantile Calculation\n", " artist_quantile, artist_quantile_dict = calculate_quantile(artist_name, cumsum_table, quantile)\n", " artist_quantile_pct, artist_quantile_pct_change_dict = calculate_quantile(artist_name, pct_table, quantile)\n", "\n", " ###### Caculate quantile and plot\n", " result = artist_data_percentile.apply(quantile_com_all_sony_artist, cal_dict=artist_quantile_dict)\n", "# fig = go.Figure()\n", " get_result(result, artist_quantile, '{}% Quantile for All Artists'.format(quantile*100), 0, quantile)\n", "# fig.update_layout(title='',\n", "# xaxis_title='Days',\n", "# yaxis_title='Streams',\n", "# plot_bgcolor='rgb(256,256,256)')\n", "\n", "# fig.show() \n", "# pct_result = artist_data_percentile_pct.apply(quantile_com_all_sony_artist, cal_dict = artist_quantile_pct_change_dict)\n", "# get_result(pct_result, artist_quantile_pct, '99% Quantile for All Artists', 0, quantile)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Current Tracks VS All History Tracks (for one artist) (Track Level)" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "def quantile_com_artist(x, cal_dict):\n", " if x.dtype != np.number:\n", " return x\n", " else:\n", " x = x.where(x>(cal_dict[x.name]))\n", " return x\n", "\n", "def calculate_quantile_artist(track_isrc, artist_quantile, quantile_level, exclude = True):\n", " artist_quantile = artist_quantile[artist_quantile.track_isrc != track_isrc]\n", " all_artist_quantile = artist_quantile.quantile(quantile_level, numeric_only=True)\n", " artist_quantile_dict = dict(zip(all_artist_quantile.index, all_artist_quantile.values))\n", " return all_artist_quantile, artist_quantile_dict\n", "\n", "\n", "def get_result_track(dataset, quantile_data, label, start_day, quantile):\n", " streams = dataset[range(start_day, 91)]\n", " streams = streams.dropna(axis = 1)\n", " if not streams.empty:\n", " if streams.shape[1] > 10:\n", " print(\"Track name: {0}; {1} days (from Day {2} to Day {3})that surpass the {4}% quantile for all tracks\".format(track_info[i], \\\n", " streams.shape[1], \\\n", " streams.columns[0], \\\n", " streams.columns[-1], \\\n", " quantile*100))\n", "# fig.add_trace(go.Scatter(x=list(streams.columns)[start_day:], \n", "# y=streams.values[0][start_day:], \n", "# name=track_info[i],\n", "# line=dict(color='green', width=3),\n", "# mode = 'lines', \n", "# connectgaps=True))\n", "# fig.add_trace(go.Scatter(x=list(range(start_day, 91)), y=quantile_data[start_day:], \\\n", "# name= label, \\\n", "# line=dict(color='blue', width=3), line_shape='spline', \\\n", "# mode = 'lines', \\\n", "# connectgaps=True,))\n", " else:\n", " print(\"There is no track that the streams is surpass the {0}% quantile for all tracks\".format(quantile*100))" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Track name: Havana; 60 days (from Day 30 to Day 90)that surpass the 99.5% quantile for all tracks\n", "Track name: Liar; 27 days (from Day 5 to Day 36)that surpass the 99.5% quantile for all tracks\n" ] } ], "source": [ "if __name__ == '__main__':\n", " \n", " artist_name = 'Camila Cabello'\n", " quantile = 0.995\n", " \n", " artist_data_percentile_old = get_artist_data(artist_name, cumsum_table)\n", " artist_data_percentile_pct_old = get_artist_data(artist_name, pct_table)\n", " artist_data_percentile = remove_duplicate(artist_data_percentile_old)\n", " artist_data_percentile_pct = remove_duplicate(artist_data_percentile_pct_old)\n", "\n", " ###### track info (dictionary{key:value})\n", " track_info = dict(zip(artist_data_percentile.track_isrc, artist_data_percentile.track_name))\n", " \n", "# fig = go.Figure()\n", " for i in set(artist_data_percentile.track_isrc):\n", " song_data = artist_data_percentile[artist_data_percentile.track_isrc == i]\n", " if song_data.shape[0]>1:\n", " trackname = song_data.track_name.iloc[0]\n", " song_data = song_data.fillna(0)\n", " song_data = song_data.groupby(['primary_artist_name', 'track_isrc', 'min_report_date']).sum().reset_index()\n", " song_data['track_name'] = trackname\n", " artist_quantile, artist_quantile_dict = calculate_quantile_artist(i, artist_data_percentile, quantile, exclude=False)\n", " percentile_result = song_data.apply(quantile_com_artist, cal_dict = artist_quantile_dict)\n", " get_result_track(percentile_result, artist_quantile, \\\n", " '{0}% Quantile for All Other Tracks except {1}'.format(quantile*100, track_info[i]), \\\n", " 5, quantile)\n", "# fig.update_layout(title='',\n", "# xaxis_title='Days',\n", "# yaxis_title='Streams',\n", "# plot_bgcolor='rgb(256,256,256)')\n", "# fig.show() " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }