{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b91a6bf-944f-4b22-8d89-5172138410f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "266b9a28-4105-4af6-b477-cbecb5da5774",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# echarts library\n",
    "!pip -q install pyecharts absl-py\n",
    "!pip -q install awswrangler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "48b01e32-c70e-495a-a4f0-629495e805be",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: fuzzy-c-means in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (1.7.0)\n",
      "Requirement already satisfied: joblib<2.0.0,>=1.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fuzzy-c-means) (1.3.1)\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fuzzy-c-means) (1.24.4)\n",
      "Requirement already satisfied: pydantic<2.0.0,>=1.9.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fuzzy-c-means) (1.10.12)\n",
      "Requirement already satisfied: tabulate<0.9.0,>=0.8.9 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fuzzy-c-means) (0.8.10)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.64.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fuzzy-c-means) (4.65.0)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from fuzzy-c-means) (0.4.2)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from pydantic<2.0.0,>=1.9.0->fuzzy-c-means) (4.7.1)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from typer<0.5.0,>=0.4.0->fuzzy-c-means) (8.1.6)\n",
      "Requirement already satisfied: kneed in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (0.8.5)\n",
      "Requirement already satisfied: numpy>=1.14.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from kneed) (1.24.4)\n",
      "Requirement already satisfied: scipy>=1.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from kneed) (1.11.1)\n",
      "Requirement already satisfied: kmodes in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (0.12.2)\n",
      "Requirement already satisfied: numpy>=1.10.4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from kmodes) (1.24.4)\n",
      "Requirement already satisfied: scikit-learn>=0.22.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from kmodes) (1.3.0)\n",
      "Requirement already satisfied: scipy>=0.13.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from kmodes) (1.11.1)\n",
      "Requirement already satisfied: joblib>=0.11 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from kmodes) (1.3.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from scikit-learn>=0.22.0->kmodes) (3.2.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:absl:READY!!!\n"
     ]
    }
   ],
   "source": [
    "!pip install fuzzy-c-means\n",
    "!pip install kneed\n",
    "!pip install kmodes\n",
    "\n",
    "import pandas as pd\n",
    "import pyecharts as echarts\n",
    "import os\n",
    "import boto3 \n",
    "import awswrangler as wr\n",
    "\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib \n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.metrics \n",
    "\n",
    "from datetime import datetime, date\n",
    "from scipy.stats import pearsonr\n",
    "from kneed import KneeLocator\n",
    "from sklearn.cluster import KMeans, AgglomerativeClustering\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler, OrdinalEncoder\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.metrics.cluster import contingency_matrix\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.model_selection import train_test_split\n",
    "from kmodes.kprototypes import KPrototypes\n",
    "from kmodes.kmodes import KModes\n",
    "\n",
    "\n",
    "# utils\n",
    "from getpass import getpass\n",
    "\n",
    "# logging and re\n",
    "from absl import logging\n",
    "import re\n",
    "\n",
    "\n",
    "log_level = \"DEBUG\"\n",
    "ticket_code = \"EXP_1\"\n",
    "\n",
    "logging.set_verbosity(log_level)\n",
    "logging.debug(\"READY!!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d8a76ac1-bc81-4d58-8d86-595f4ad2c3dc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sec_id = 'dev/sagemaker-notebook-instance/SNOWFLAKE_PASSWORD'\n",
    "\n",
    "\n",
    "def get_secret_value(name, version=None):\n",
    "    \"\"\"Gets the value of a secret.\n",
    "\n",
    "    Version (if defined) is used to retrieve a particular version of\n",
    "    the secret.\n",
    "\n",
    "    \"\"\"\n",
    "    secrets_client = boto3.client(\"secretsmanager\")\n",
    "    kwargs = {'SecretId': name}\n",
    "    if version is not None:\n",
    "        kwargs['VersionStage'] = version\n",
    "    response = secrets_client.get_secret_value(**kwargs)\n",
    "    return response\n",
    "\n",
    "\n",
    "def get_snowflake_creds(username=\"SAGEMAKER\", account=\"orchard\",\n",
    "                        warehouse=\"DEV_OWS_ENGINEERING\"):\n",
    "    \"\"\"\n",
    "    Fetches and returns snowflake creds for connecting to snowflake\n",
    "\n",
    "    Please use this within the scope of a function if using this on a shared instance\n",
    "    This is so that the password is in memory only when its needed and gets dropped \n",
    "    once its no longer required.\n",
    "\n",
    "    returns:\n",
    "    - creds (dict) - a dictionary containing user creds\n",
    "\n",
    "    \"\"\"\n",
    "    creds = {\n",
    "      \"user\":  username,\n",
    "      \"password\": get_secret_value(sec_id)['SecretString'],\n",
    "      \"account\": \"orchard\",\n",
    "      \"warehouse\": warehouse,\n",
    "      \"protocol\": 'https'\n",
    "    }\n",
    "    return creds\n",
    "\n",
    "\n",
    "def snowflake_connector_factory(creds=None):\n",
    "    \"\"\"\n",
    "    A Factory for creating snowflake connectors.\n",
    "\n",
    "    This returns the cursor after opening a session with snowflake.\n",
    "\n",
    "    params:\n",
    "    - creds - snowflake credentials \n",
    "\n",
    "    returns:\n",
    "    - cursor - snowflake session cursor\n",
    "    \"\"\"\n",
    "    try:\n",
    "        if creds:\n",
    "            _creds = creds\n",
    "        else:\n",
    "            _creds = get_snowflake_creds()\n",
    "        return snowflake.connector.connect(**_creds).cursor()\n",
    "    except Exception as e:\n",
    "        logging.error(f\"Something went wrong - {str(e)}\")\n",
    "\n",
    "\n",
    "def _is_version_number(s):\n",
    "    \"Check and returns true if its a version number\"\n",
    "    return re.search(\"^[0-9][.0-9]*[0-9]$\", s) is not None\n",
    "\n",
    "\n",
    "def test_connection():\n",
    "    \"\"\" tests connection to snowflake \"\"\"\n",
    "    with snowflake_connector_factory() as cs:\n",
    "        try:\n",
    "            cs.execute(\"SELECT current_version()\")\n",
    "            one_row = cs.fetchone()\n",
    "            assert len(one_row) == 1\n",
    "            assert _is_version_number(one_row[0])\n",
    "            logging.info(f\"Your snowflake version - {one_row[0]} PASSED!\")\n",
    "        except Exception as e:\n",
    "          logging.error(f\"Something went wrong - {str(e)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1781ce54-081a-455e-9d9c-9a6f6a942021",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:absl:Your snowflake version - 7.29.1 PASSED!\n"
     ]
    }
   ],
   "source": [
    "test_connection()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cfbd460-c260-45fa-9efc-9a67ba8defcc",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce4500ee-d33f-40ff-bcba-0d429b10c1ab",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "668dbda4-18c0-4071-bab3-d32401ad0b55",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with snowflake_connector_factory() as cs:\n",
    "    try:\n",
    "        cs.execute(\"USE WAREHOUSE DEV_PERFORMANCE_WAREHOUSE;\")\n",
    "        cs.execute(\"\"\"\n",
    "        \n",
    "        select * from DEV_ENGINEERING.EIMPARA.MOMENTS_140DAYS_K_PROTO_V3;\n",
    "        \"\"\")\n",
    "        rows = cs.fetchall()\n",
    "    except Exception as e:\n",
    "      logging.error(f\"Something went wrong - {str(e)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7cecf916-229a-4950-9705-c37e6f94be0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(140000, 7)\n",
      "(140000, 7)\n"
     ]
    }
   ],
   "source": [
    "data_df = pd.DataFrame(rows, columns=map(lambda meta: meta[0], cs.description))\n",
    "df = data_df.drop_duplicates().copy()\n",
    "print(data_df.shape)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f299f18c-d5e6-4990-a916-30f7bd9ef07e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# with snowflake_connector_factory() as cs:\n",
    "#     try:\n",
    "#         cs.execute(\"USE WAREHOUSE DEV_PERFORMANCE_WAREHOUSE;\")\n",
    "#         cs.execute(\"\"\"\n",
    "        \n",
    "#         select * from select * from DEV_ENGINEERING.EIMPARA.MOMENTS_FOURIERTABLE_APR_SEPT_22_V2;\n",
    "#         \"\"\")\n",
    "#         inf_rows = cs.fetchall()\n",
    "#     except Exception as e:\n",
    "#       logging.error(f\"Something went wrong - {str(e)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21a8f663-e609-4d11-8953-5eb7e2437ed7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_df2 = pd.DataFrame(inf_rows, columns=map(lambda meta: meta[0], cs.description))\n",
    "# if_df = data_df2.drop_duplicates().copy()\n",
    "# print(data_df2.shape)\n",
    "# print(if_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0b167d59-9f80-4346-82c6-d97b05ae7783",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TRANSACTION_COUNTRY_CODE</th>\n",
       "      <th>ISRC</th>\n",
       "      <th>ARTISTID</th>\n",
       "      <th>UPC</th>\n",
       "      <th>RELEASENAME</th>\n",
       "      <th>RELEASEDATE</th>\n",
       "      <th>SALE_START_DATE</th>\n",
       "      <th>NEW_RELEASE</th>\n",
       "      <th>FORMAT</th>\n",
       "      <th>COMPILATION</th>\n",
       "      <th>GENREID</th>\n",
       "      <th>LABELID</th>\n",
       "      <th>ARTIST_OLDEST_RELEASE_DATE</th>\n",
       "      <th>ARTIST_RELEASES_SINCE_TRACK_IN_QUESTION_RELEASE</th>\n",
       "      <th>ARTIST_RELEASE_RATE_SINCE_TRACK_IN_QUESTION_RELEASE</th>\n",
       "      <th>ACTIVE_STREAMS</th>\n",
       "      <th>COLLECTION_STREAMS</th>\n",
       "      <th>OTHER_STREAMS</th>\n",
       "      <th>PASSIVE_STREAMS</th>\n",
       "      <th>STREAMS</th>\n",
       "      <th>AVERAGE_DAILY_STREAMS</th>\n",
       "      <th>AVG_FE</th>\n",
       "      <th>AVG_MAE</th>\n",
       "      <th>AVG_RMSE</th>\n",
       "      <th>FIRST_DAY_TRAIN_SET_STREAMS</th>\n",
       "      <th>LAST_DAY_TRAIN_SET_STREAMS</th>\n",
       "      <th>LEN_DF</th>\n",
       "      <th>LINEAR_GRADIENT</th>\n",
       "      <th>MAE</th>\n",
       "      <th>MEDIAN_STREAMS</th>\n",
       "      <th>MSE</th>\n",
       "      <th>NRMSE</th>\n",
       "      <th>PRED_TYPE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>SUM_FORECAST_ERRORS</th>\n",
       "      <th>TEST_DATA_LINEAR_GRADIENT</th>\n",
       "      <th>TEST_DATA_MAX_DATE_STREAMS</th>\n",
       "      <th>TEST_DATA_MIN_DATE_STREAMS</th>\n",
       "      <th>DANCEABILITY</th>\n",
       "      <th>ACOUSTICNESS</th>\n",
       "      <th>LIVENESS</th>\n",
       "      <th>LOUDNESS</th>\n",
       "      <th>DURATION_MS</th>\n",
       "      <th>ENERGY</th>\n",
       "      <th>INSTRUMENTALNESS</th>\n",
       "      <th>KEY</th>\n",
       "      <th>MODE</th>\n",
       "      <th>SPEECHINESS</th>\n",
       "      <th>TEMPO</th>\n",
       "      <th>TIME_SIGNATURE</th>\n",
       "      <th>VALENCE</th>\n",
       "      <th>sign</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GB</td>\n",
       "      <td>QM4TX1913874</td>\n",
       "      <td>1011137</td>\n",
       "      <td>193483748719</td>\n",
       "      <td>City of Skin and Bone</td>\n",
       "      <td>2019-11-15</td>\n",
       "      <td>2019-11-15</td>\n",
       "      <td>New</td>\n",
       "      <td>Full Length</td>\n",
       "      <td>N</td>\n",
       "      <td>1</td>\n",
       "      <td>26681</td>\n",
       "      <td>2018-10-05</td>\n",
       "      <td>11</td>\n",
       "      <td>0.323529</td>\n",
       "      <td>72</td>\n",
       "      <td>2345</td>\n",
       "      <td>231</td>\n",
       "      <td>6619</td>\n",
       "      <td>9267</td>\n",
       "      <td>66.864662</td>\n",
       "      <td>-0.007600</td>\n",
       "      <td>0.104311</td>\n",
       "      <td>0.075747</td>\n",
       "      <td>87</td>\n",
       "      <td>65</td>\n",
       "      <td>140</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>14.603471</td>\n",
       "      <td>63.0</td>\n",
       "      <td>112.458356</td>\n",
       "      <td>0.246619</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>10.604638</td>\n",
       "      <td>-1.064</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>61</td>\n",
       "      <td>56</td>\n",
       "      <td>0.490</td>\n",
       "      <td>0.000496</td>\n",
       "      <td>0.0899</td>\n",
       "      <td>-9.050</td>\n",
       "      <td>206053</td>\n",
       "      <td>0.492</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0310</td>\n",
       "      <td>157.954</td>\n",
       "      <td>4</td>\n",
       "      <td>0.541</td>\n",
       "      <td>overall_negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GB</td>\n",
       "      <td>QMBZ91955892</td>\n",
       "      <td>1670892</td>\n",
       "      <td>194491532789</td>\n",
       "      <td>A N T I T H E S I S, Vol. 1, Pt. 2</td>\n",
       "      <td>2019-12-13</td>\n",
       "      <td>2019-12-13</td>\n",
       "      <td></td>\n",
       "      <td>Full Length</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>32600</td>\n",
       "      <td>2019-12-06</td>\n",
       "      <td>27</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>118</td>\n",
       "      <td>2175</td>\n",
       "      <td>168</td>\n",
       "      <td>6801</td>\n",
       "      <td>9262</td>\n",
       "      <td>66.105263</td>\n",
       "      <td>0.296157</td>\n",
       "      <td>0.100465</td>\n",
       "      <td>0.093855</td>\n",
       "      <td>76</td>\n",
       "      <td>65</td>\n",
       "      <td>140</td>\n",
       "      <td>-0.083333</td>\n",
       "      <td>14.065038</td>\n",
       "      <td>63.0</td>\n",
       "      <td>172.651976</td>\n",
       "      <td>0.355127</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>13.139710</td>\n",
       "      <td>41.462</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "      <td>0.636</td>\n",
       "      <td>0.691000</td>\n",
       "      <td>0.2160</td>\n",
       "      <td>-9.551</td>\n",
       "      <td>123452</td>\n",
       "      <td>0.241</td>\n",
       "      <td>0.793000</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0564</td>\n",
       "      <td>75.959</td>\n",
       "      <td>4</td>\n",
       "      <td>0.256</td>\n",
       "      <td>overall_positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GB</td>\n",
       "      <td>ARF411900277</td>\n",
       "      <td>660182</td>\n",
       "      <td>194491500504</td>\n",
       "      <td>The Real Sound of Style</td>\n",
       "      <td>2019-10-25</td>\n",
       "      <td>2019-10-25</td>\n",
       "      <td></td>\n",
       "      <td>Full Length</td>\n",
       "      <td>N</td>\n",
       "      <td>2</td>\n",
       "      <td>8099</td>\n",
       "      <td>2014-04-29</td>\n",
       "      <td>11</td>\n",
       "      <td>0.314286</td>\n",
       "      <td>156</td>\n",
       "      <td>1623</td>\n",
       "      <td>200</td>\n",
       "      <td>7282</td>\n",
       "      <td>9261</td>\n",
       "      <td>66.345865</td>\n",
       "      <td>-0.393921</td>\n",
       "      <td>0.040850</td>\n",
       "      <td>0.083940</td>\n",
       "      <td>61</td>\n",
       "      <td>76</td>\n",
       "      <td>140</td>\n",
       "      <td>0.113636</td>\n",
       "      <td>5.719023</td>\n",
       "      <td>66.0</td>\n",
       "      <td>138.099771</td>\n",
       "      <td>0.273293</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>11.751586</td>\n",
       "      <td>-55.149</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>73</td>\n",
       "      <td>67</td>\n",
       "      <td>0.765</td>\n",
       "      <td>0.006220</td>\n",
       "      <td>0.0434</td>\n",
       "      <td>-14.624</td>\n",
       "      <td>385500</td>\n",
       "      <td>0.383</td>\n",
       "      <td>0.078100</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0811</td>\n",
       "      <td>109.992</td>\n",
       "      <td>4</td>\n",
       "      <td>0.690</td>\n",
       "      <td>overall_negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GB</td>\n",
       "      <td>GBAYE9701654</td>\n",
       "      <td>2745111</td>\n",
       "      <td>5054526143395</td>\n",
       "      <td>My Way</td>\n",
       "      <td>1966-01-01</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td></td>\n",
       "      <td>Full Length</td>\n",
       "      <td>N</td>\n",
       "      <td>7</td>\n",
       "      <td>46787</td>\n",
       "      <td>1966-01-01</td>\n",
       "      <td>7</td>\n",
       "      <td>0.010294</td>\n",
       "      <td>226</td>\n",
       "      <td>1959</td>\n",
       "      <td>681</td>\n",
       "      <td>6393</td>\n",
       "      <td>9259</td>\n",
       "      <td>65.285714</td>\n",
       "      <td>-0.218571</td>\n",
       "      <td>0.381466</td>\n",
       "      <td>0.043313</td>\n",
       "      <td>54</td>\n",
       "      <td>85</td>\n",
       "      <td>140</td>\n",
       "      <td>0.234848</td>\n",
       "      <td>53.405182</td>\n",
       "      <td>56.0</td>\n",
       "      <td>36.769395</td>\n",
       "      <td>0.606378</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>6.063777</td>\n",
       "      <td>-30.600</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>85</td>\n",
       "      <td>82</td>\n",
       "      <td>0.458</td>\n",
       "      <td>0.275000</td>\n",
       "      <td>0.0673</td>\n",
       "      <td>-8.408</td>\n",
       "      <td>137627</td>\n",
       "      <td>0.637</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0301</td>\n",
       "      <td>76.276</td>\n",
       "      <td>4</td>\n",
       "      <td>0.858</td>\n",
       "      <td>overall_negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GB</td>\n",
       "      <td>GBKPL2030040</td>\n",
       "      <td>2748056</td>\n",
       "      <td>5054526961784</td>\n",
       "      <td>Welcome to the Hills</td>\n",
       "      <td>2020-12-03</td>\n",
       "      <td>2020-12-03</td>\n",
       "      <td></td>\n",
       "      <td>Full Length</td>\n",
       "      <td>N</td>\n",
       "      <td>8</td>\n",
       "      <td>43662</td>\n",
       "      <td>2018-07-11</td>\n",
       "      <td>16</td>\n",
       "      <td>0.761905</td>\n",
       "      <td>2901</td>\n",
       "      <td>3267</td>\n",
       "      <td>414</td>\n",
       "      <td>2676</td>\n",
       "      <td>9258</td>\n",
       "      <td>66.203008</td>\n",
       "      <td>-0.028743</td>\n",
       "      <td>0.011322</td>\n",
       "      <td>0.057458</td>\n",
       "      <td>76</td>\n",
       "      <td>72</td>\n",
       "      <td>140</td>\n",
       "      <td>-0.030303</td>\n",
       "      <td>1.585039</td>\n",
       "      <td>64.0</td>\n",
       "      <td>64.708627</td>\n",
       "      <td>0.383056</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>8.044167</td>\n",
       "      <td>-4.024</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>73</td>\n",
       "      <td>69</td>\n",
       "      <td>0.303</td>\n",
       "      <td>0.368000</td>\n",
       "      <td>0.3160</td>\n",
       "      <td>-13.742</td>\n",
       "      <td>441613</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.844000</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0523</td>\n",
       "      <td>166.914</td>\n",
       "      <td>4</td>\n",
       "      <td>0.315</td>\n",
       "      <td>overall_negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  TRANSACTION_COUNTRY_CODE          ISRC  ARTISTID            UPC  \\\n",
       "0                       GB  QM4TX1913874   1011137   193483748719   \n",
       "1                       GB  QMBZ91955892   1670892   194491532789   \n",
       "2                       GB  ARF411900277    660182   194491500504   \n",
       "3                       GB  GBAYE9701654   2745111  5054526143395   \n",
       "4                       GB  GBKPL2030040   2748056  5054526961784   \n",
       "\n",
       "                          RELEASENAME RELEASEDATE SALE_START_DATE NEW_RELEASE  \\\n",
       "0               City of Skin and Bone  2019-11-15      2019-11-15         New   \n",
       "1  A N T I T H E S I S, Vol. 1, Pt. 2  2019-12-13      2019-12-13               \n",
       "2             The Real Sound of Style  2019-10-25      2019-10-25               \n",
       "3                              My Way  1966-01-01      2017-01-15               \n",
       "4                Welcome to the Hills  2020-12-03      2020-12-03               \n",
       "\n",
       "        FORMAT COMPILATION  GENREID  LABELID ARTIST_OLDEST_RELEASE_DATE  \\\n",
       "0  Full Length           N        1    26681                 2018-10-05   \n",
       "1  Full Length           N        6    32600                 2019-12-06   \n",
       "2  Full Length           N        2     8099                 2014-04-29   \n",
       "3  Full Length           N        7    46787                 1966-01-01   \n",
       "4  Full Length           N        8    43662                 2018-07-11   \n",
       "\n",
       "   ARTIST_RELEASES_SINCE_TRACK_IN_QUESTION_RELEASE  \\\n",
       "0                                               11   \n",
       "1                                               27   \n",
       "2                                               11   \n",
       "3                                                7   \n",
       "4                                               16   \n",
       "\n",
       "   ARTIST_RELEASE_RATE_SINCE_TRACK_IN_QUESTION_RELEASE  ACTIVE_STREAMS  \\\n",
       "0                                           0.323529                72   \n",
       "1                                           0.818182               118   \n",
       "2                                           0.314286               156   \n",
       "3                                           0.010294               226   \n",
       "4                                           0.761905              2901   \n",
       "\n",
       "   COLLECTION_STREAMS  OTHER_STREAMS  PASSIVE_STREAMS  STREAMS  \\\n",
       "0                2345            231             6619     9267   \n",
       "1                2175            168             6801     9262   \n",
       "2                1623            200             7282     9261   \n",
       "3                1959            681             6393     9259   \n",
       "4                3267            414             2676     9258   \n",
       "\n",
       "   AVERAGE_DAILY_STREAMS    AVG_FE   AVG_MAE  AVG_RMSE  \\\n",
       "0              66.864662 -0.007600  0.104311  0.075747   \n",
       "1              66.105263  0.296157  0.100465  0.093855   \n",
       "2              66.345865 -0.393921  0.040850  0.083940   \n",
       "3              65.285714 -0.218571  0.381466  0.043313   \n",
       "4              66.203008 -0.028743  0.011322  0.057458   \n",
       "\n",
       "   FIRST_DAY_TRAIN_SET_STREAMS  LAST_DAY_TRAIN_SET_STREAMS  LEN_DF  \\\n",
       "0                           87                          65     140   \n",
       "1                           76                          65     140   \n",
       "2                           61                          76     140   \n",
       "3                           54                          85     140   \n",
       "4                           76                          72     140   \n",
       "\n",
       "   LINEAR_GRADIENT        MAE  MEDIAN_STREAMS         MSE     NRMSE  \\\n",
       "0        -0.166667  14.603471            63.0  112.458356  0.246619   \n",
       "1        -0.083333  14.065038            63.0  172.651976  0.355127   \n",
       "2         0.113636   5.719023            66.0  138.099771  0.273293   \n",
       "3         0.234848  53.405182            56.0   36.769395  0.606378   \n",
       "4        -0.030303   1.585039            64.0   64.708627  0.383056   \n",
       "\n",
       "                    PRED_TYPE       RMSE  SUM_FORECAST_ERRORS  \\\n",
       "0  actuals_crossing_predicted  10.604638               -1.064   \n",
       "1  actuals_crossing_predicted  13.139710               41.462   \n",
       "2  actuals_crossing_predicted  11.751586              -55.149   \n",
       "3  actuals_crossing_predicted   6.063777              -30.600   \n",
       "4  actuals_crossing_predicted   8.044167               -4.024   \n",
       "\n",
       "   TEST_DATA_LINEAR_GRADIENT  TEST_DATA_MAX_DATE_STREAMS  \\\n",
       "0                   0.714286                          61   \n",
       "1                   0.000000                          78   \n",
       "2                   0.857143                          73   \n",
       "3                   0.428571                          85   \n",
       "4                   0.571429                          73   \n",
       "\n",
       "   TEST_DATA_MIN_DATE_STREAMS  DANCEABILITY  ACOUSTICNESS  LIVENESS  LOUDNESS  \\\n",
       "0                          56         0.490      0.000496    0.0899    -9.050   \n",
       "1                          78         0.636      0.691000    0.2160    -9.551   \n",
       "2                          67         0.765      0.006220    0.0434   -14.624   \n",
       "3                          82         0.458      0.275000    0.0673    -8.408   \n",
       "4                          69         0.303      0.368000    0.3160   -13.742   \n",
       "\n",
       "   DURATION_MS  ENERGY  INSTRUMENTALNESS  KEY  MODE  SPEECHINESS    TEMPO  \\\n",
       "0       206053   0.492          0.000022    0     1       0.0310  157.954   \n",
       "1       123452   0.241          0.793000    7     1       0.0564   75.959   \n",
       "2       385500   0.383          0.078100    0     1       0.0811  109.992   \n",
       "3       137627   0.637          0.000000    5     1       0.0301   76.276   \n",
       "4       441613   0.632          0.844000   11     0       0.0523  166.914   \n",
       "\n",
       "   TIME_SIGNATURE  VALENCE              sign  \n",
       "0               4    0.541  overall_negative  \n",
       "1               4    0.256  overall_positive  \n",
       "2               4    0.690  overall_negative  \n",
       "3               4    0.858  overall_negative  \n",
       "4               4    0.315  overall_negative  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sign'] = np.where(df['SUM_FORECAST_ERRORS'] > 0, 'overall_positive', 'overall_negative')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4e8eac56-8fa2-4e5c-8ccf-f4b30143e2c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "N    10849\n",
       "Y       44\n",
       "Name: COMPILATION, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['COMPILATION'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e0fdbe7d-785e-4edf-8309-cd834f0b0145",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "actuals_crossing_predicted    9306\n",
       "actuals_below_predicted       1049\n",
       "actuals_above_predicted        538\n",
       "Name: PRED_TYPE, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['PRED_TYPE'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "27946d58-be52-405a-8deb-e8be16c1759f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10893 entries, 0 to 10892\n",
      "Data columns (total 53 columns):\n",
      " #   Column                                               Non-Null Count  Dtype         \n",
      "---  ------                                               --------------  -----         \n",
      " 0   TRANSACTION_COUNTRY_CODE                             10893 non-null  object        \n",
      " 1   ISRC                                                 10893 non-null  object        \n",
      " 2   ARTISTID                                             10893 non-null  int32         \n",
      " 3   UPC                                                  10893 non-null  int64         \n",
      " 4   RELEASENAME                                          10893 non-null  object        \n",
      " 5   RELEASEDATE                                          10805 non-null  datetime64[ns]\n",
      " 6   SALE_START_DATE                                      10809 non-null  datetime64[ns]\n",
      " 7   NEW_RELEASE                                          10893 non-null  object        \n",
      " 8   FORMAT                                               9908 non-null   object        \n",
      " 9   COMPILATION                                          10893 non-null  object        \n",
      " 10  GENREID                                              10893 non-null  int8          \n",
      " 11  LABELID                                              10893 non-null  int32         \n",
      " 12  ARTIST_OLDEST_RELEASE_DATE                           10887 non-null  datetime64[ns]\n",
      " 13  ARTIST_RELEASES_SINCE_TRACK_IN_QUESTION_RELEASE      10893 non-null  int16         \n",
      " 14  ARTIST_RELEASE_RATE_SINCE_TRACK_IN_QUESTION_RELEASE  10893 non-null  float64       \n",
      " 15  ACTIVE_STREAMS                                       10893 non-null  int32         \n",
      " 16  COLLECTION_STREAMS                                   10893 non-null  int32         \n",
      " 17  OTHER_STREAMS                                        10893 non-null  int32         \n",
      " 18  PASSIVE_STREAMS                                      10893 non-null  int32         \n",
      " 19  STREAMS                                              10893 non-null  int32         \n",
      " 20  AVERAGE_DAILY_STREAMS                                10893 non-null  float64       \n",
      " 21  AVG_FE                                               10893 non-null  float64       \n",
      " 22  AVG_MAE                                              10893 non-null  float64       \n",
      " 23  AVG_RMSE                                             10893 non-null  float64       \n",
      " 24  FIRST_DAY_TRAIN_SET_STREAMS                          10893 non-null  int32         \n",
      " 25  LAST_DAY_TRAIN_SET_STREAMS                           10893 non-null  int32         \n",
      " 26  LEN_DF                                               10893 non-null  int16         \n",
      " 27  LINEAR_GRADIENT                                      10893 non-null  float64       \n",
      " 28  MAE                                                  10893 non-null  float64       \n",
      " 29  MEDIAN_STREAMS                                       10893 non-null  float64       \n",
      " 30  MSE                                                  10893 non-null  float64       \n",
      " 31  NRMSE                                                10893 non-null  float64       \n",
      " 32  PRED_TYPE                                            10893 non-null  object        \n",
      " 33  RMSE                                                 10893 non-null  float64       \n",
      " 34  SUM_FORECAST_ERRORS                                  10893 non-null  float64       \n",
      " 35  TEST_DATA_LINEAR_GRADIENT                            10893 non-null  float64       \n",
      " 36  TEST_DATA_MAX_DATE_STREAMS                           10893 non-null  int32         \n",
      " 37  TEST_DATA_MIN_DATE_STREAMS                           10893 non-null  int32         \n",
      " 38  DANCEABILITY                                         10893 non-null  float64       \n",
      " 39  ACOUSTICNESS                                         10893 non-null  float64       \n",
      " 40  LIVENESS                                             10893 non-null  float64       \n",
      " 41  LOUDNESS                                             10893 non-null  float64       \n",
      " 42  DURATION_MS                                          10893 non-null  int32         \n",
      " 43  ENERGY                                               10893 non-null  float64       \n",
      " 44  INSTRUMENTALNESS                                     10893 non-null  float64       \n",
      " 45  KEY                                                  10893 non-null  object        \n",
      " 46  MODE                                                 10893 non-null  object        \n",
      " 47  SPEECHINESS                                          10893 non-null  float64       \n",
      " 48  TEMPO                                                10893 non-null  float64       \n",
      " 49  TIME_SIGNATURE                                       10893 non-null  object        \n",
      " 50  VALENCE                                              10893 non-null  float64       \n",
      " 51  sign                                                 10893 non-null  object        \n",
      " 52  Genre_recoded                                        10893 non-null  object        \n",
      "dtypes: datetime64[ns](3), float64(22), int16(2), int32(12), int64(1), int8(1), object(12)\n",
      "memory usage: 3.7+ MB\n"
     ]
    }
   ],
   "source": [
    "conditions = [(df['GENREID']==1),\n",
    "              (df['GENREID']==16),\n",
    "              (df['GENREID']==27),\n",
    "              (df['GENREID']==18),\n",
    "              (df['GENREID']==4),\n",
    "              (df['GENREID']==3),\n",
    "              (df['GENREID']==24),\n",
    "              (df['GENREID']==19),\n",
    "              (df['GENREID']==7),\n",
    "              (df['GENREID']==13),\n",
    "              (df['GENREID']==11),\n",
    "              (df['GENREID']==9),\n",
    "              (df['GENREID']==28),\n",
    "              (df['GENREID']==22),\n",
    "              (df['GENREID']==6),\n",
    "              (df['GENREID']==25),\n",
    "              (df['GENREID']==8),\n",
    "              (df['GENREID']==2),\n",
    "              (df['GENREID']==21),\n",
    "              (df['GENREID']==12),\n",
    "              (df['GENREID']==17),\n",
    "              (df['GENREID']==5),\n",
    "              (df['GENREID']==20),\n",
    "              (df['GENREID']==14),\n",
    "              (df['GENREID']==15)]\n",
    "\n",
    "choices = ['Rock', 'Spoken Word', 'Soundtracks', 'Latin Music', 'Punk', 'Metal', 'Educational', 'Christian', 'Pop', \n",
    "           \"Children's\", 'World Music', 'Country', 'Alternative', 'DVD', 'Hip-hop/Rap', 'R&B', 'Jazz', 'Electronic',\n",
    "          'Reggae', 'Classical', 'Video', 'Folk', 'Blues', 'New Age', 'Holiday']\n",
    "\n",
    "\n",
    "df['Genre_recoded'] = np.select(conditions, choices)\n",
    "\n",
    "df['Genre_recoded'] = df['Genre_recoded'].astype(str)\n",
    "df['KEY'] = df['KEY'].astype(str)\n",
    "df['MODE'] = df['MODE'].astype(str)\n",
    "df['TIME_SIGNATURE'] = df['TIME_SIGNATURE'].astype(str)\n",
    "\n",
    "df['ARTIST_RELEASE_RATE_SINCE_TRACK_IN_QUESTION_RELEASE'] = df['ARTIST_RELEASE_RATE_SINCE_TRACK_IN_QUESTION_RELEASE'].fillna(0)\n",
    "\n",
    "df.info()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bc640201-6068-4b87-8551-d25073ad07c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Rock           2328\n",
       "Pop            1689\n",
       "Hip-hop/Rap    1252\n",
       "Electronic     1217\n",
       "World Music     848\n",
       "Metal           542\n",
       "R&B             411\n",
       "Soundtracks     337\n",
       "Children's      306\n",
       "Folk            300\n",
       "Jazz            262\n",
       "Latin Music     254\n",
       "Country         248\n",
       "Reggae          223\n",
       "Punk            191\n",
       "New Age         173\n",
       "Classical       116\n",
       "Christian        85\n",
       "Blues            55\n",
       "Spoken Word      50\n",
       "Video             3\n",
       "Holiday           3\n",
       "Name: Genre_recoded, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Genre_recoded'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2787d967-c19d-4ede-8090-51b4485ffbea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DANCEABILITY</th>\n",
       "      <th>ACOUSTICNESS</th>\n",
       "      <th>LIVENESS</th>\n",
       "      <th>LOUDNESS</th>\n",
       "      <th>DURATION_MS</th>\n",
       "      <th>ENERGY</th>\n",
       "      <th>INSTRUMENTALNESS</th>\n",
       "      <th>SPEECHINESS</th>\n",
       "      <th>TEMPO</th>\n",
       "      <th>VALENCE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10893.000000</td>\n",
       "      <td>10893.000000</td>\n",
       "      <td>10893.000000</td>\n",
       "      <td>10893.000000</td>\n",
       "      <td>1.089300e+04</td>\n",
       "      <td>10893.000000</td>\n",
       "      <td>10893.000000</td>\n",
       "      <td>10893.000000</td>\n",
       "      <td>10893.000000</td>\n",
       "      <td>10893.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.567767</td>\n",
       "      <td>0.318392</td>\n",
       "      <td>0.190492</td>\n",
       "      <td>-8.763575</td>\n",
       "      <td>2.289916e+05</td>\n",
       "      <td>0.605818</td>\n",
       "      <td>0.170052</td>\n",
       "      <td>0.091304</td>\n",
       "      <td>120.752326</td>\n",
       "      <td>0.459819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.182795</td>\n",
       "      <td>0.331868</td>\n",
       "      <td>0.160664</td>\n",
       "      <td>5.032470</td>\n",
       "      <td>1.322087e+05</td>\n",
       "      <td>0.253683</td>\n",
       "      <td>0.314450</td>\n",
       "      <td>0.103580</td>\n",
       "      <td>29.978844</td>\n",
       "      <td>0.256648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.015500</td>\n",
       "      <td>-44.406000</td>\n",
       "      <td>3.014500e+04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.451000</td>\n",
       "      <td>0.019600</td>\n",
       "      <td>0.096700</td>\n",
       "      <td>-10.607000</td>\n",
       "      <td>1.752400e+05</td>\n",
       "      <td>0.427000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.036100</td>\n",
       "      <td>97.707000</td>\n",
       "      <td>0.247000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.581000</td>\n",
       "      <td>0.179000</td>\n",
       "      <td>0.124000</td>\n",
       "      <td>-7.538000</td>\n",
       "      <td>2.100400e+05</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.000259</td>\n",
       "      <td>0.049800</td>\n",
       "      <td>119.998000</td>\n",
       "      <td>0.445000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.705000</td>\n",
       "      <td>0.601000</td>\n",
       "      <td>0.236000</td>\n",
       "      <td>-5.469000</td>\n",
       "      <td>2.538800e+05</td>\n",
       "      <td>0.817000</td>\n",
       "      <td>0.117000</td>\n",
       "      <td>0.093600</td>\n",
       "      <td>139.899000</td>\n",
       "      <td>0.662000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.986000</td>\n",
       "      <td>0.996000</td>\n",
       "      <td>0.998000</td>\n",
       "      <td>2.704000</td>\n",
       "      <td>5.403500e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.955000</td>\n",
       "      <td>235.913000</td>\n",
       "      <td>0.991000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       DANCEABILITY  ACOUSTICNESS      LIVENESS      LOUDNESS   DURATION_MS  \\\n",
       "count  10893.000000  10893.000000  10893.000000  10893.000000  1.089300e+04   \n",
       "mean       0.567767      0.318392      0.190492     -8.763575  2.289916e+05   \n",
       "std        0.182795      0.331868      0.160664      5.032470  1.322087e+05   \n",
       "min        0.000000      0.000000      0.015500    -44.406000  3.014500e+04   \n",
       "25%        0.451000      0.019600      0.096700    -10.607000  1.752400e+05   \n",
       "50%        0.581000      0.179000      0.124000     -7.538000  2.100400e+05   \n",
       "75%        0.705000      0.601000      0.236000     -5.469000  2.538800e+05   \n",
       "max        0.986000      0.996000      0.998000      2.704000  5.403500e+06   \n",
       "\n",
       "             ENERGY  INSTRUMENTALNESS   SPEECHINESS         TEMPO  \\\n",
       "count  10893.000000      10893.000000  10893.000000  10893.000000   \n",
       "mean       0.605818          0.170052      0.091304    120.752326   \n",
       "std        0.253683          0.314450      0.103580     29.978844   \n",
       "min        0.000000          0.000000      0.000000      0.000000   \n",
       "25%        0.427000          0.000000      0.036100     97.707000   \n",
       "50%        0.640000          0.000259      0.049800    119.998000   \n",
       "75%        0.817000          0.117000      0.093600    139.899000   \n",
       "max        1.000000          1.000000      0.955000    235.913000   \n",
       "\n",
       "            VALENCE  \n",
       "count  10893.000000  \n",
       "mean       0.459819  \n",
       "std        0.256648  \n",
       "min        0.000000  \n",
       "25%        0.247000  \n",
       "50%        0.445000  \n",
       "75%        0.662000  \n",
       "max        0.991000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['DANCEABILITY', 'ACOUSTICNESS','LIVENESS', 'LOUDNESS', 'DURATION_MS', 'ENERGY', 'INSTRUMENTALNESS', 'SPEECHINESS', 'TEMPO', 'VALENCE']].describe()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0994a4c7-c973-443c-942e-b673ff772f1e",
   "metadata": {},
   "source": [
    "### Inf Point subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "db0682a7-65df-4d7a-bdfc-7d327d74af68",
   "metadata": {},
   "outputs": [],
   "source": [
    "if_df2 = if_df.copy()\n",
    "if_df2['ACTIVITY_DATE'] = pd.to_datetime(if_df2['ACTIVITY_DATE'])\n",
    "if_df2['ACTIVITY_DATE'] = if_df2['ACTIVITY_DATE'].dt.date\n",
    "if_df2 = if_df2[if_df2['ACTIVITY_DATE'] > date(2022, 9, 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4837bbcf-dd86-4b75-89e1-e0d7e32fcdc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    7033806\n",
       "1     191594\n",
       "Name: INFLECTION_POINT, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if_df['INFLECTION_POINT'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "732c8dd8-a4bb-4e34-987e-6dc1acc03fe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "if_sub = if_df2[if_df2['INFLECTION_POINT']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d6e229b4-3dea-411a-84aa-4f9263531aa1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10901"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if_sub['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8fb02804-ab94-476d-a2e0-b10bb03544c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "inflection_list_id = if_sub['ISRC'].unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "38cc7e65-a422-4772-8f96-7f6b9a4cafd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df[df['ISRC'].isin(inflection_list_id)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "fc3d2522-b8f4-47e2-b1d0-076a0b4f1c27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10893, 53)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "14c27cbc-c025-4dc9-9cb6-a6f3f0356417",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(list(df1.columns))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d84d092-9b14-47a3-8202-240c37efb91f",
   "metadata": {},
   "source": [
    "# Working on subset of overall positive sign "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "58bbca18-61fc-4116-a914-5d86680784a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df1 = original.copy()\n",
    "sub = df1[(df1['ISRC'].isin(inflection_list_id)) & (df1['sign']=='overall_positive')].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8237b085-7f41-450c-b008-f6103f8aa36e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4739, 53)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d5a35bce-bf4e-45b8-aa1b-be686b6d702f",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_bins = 3\n",
    "sub['DANCEABILITY_categorical'] = pd.cut(sub['DANCEABILITY'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['ACOUSTICNESS_categorical'] = pd.cut(sub['ACOUSTICNESS'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['LIVENESS_categorical'] = pd.cut(sub['LIVENESS'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['LOUDNESS_categorical'] = pd.cut(sub['LOUDNESS'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['DURATION_MS_categorical'] = pd.cut(sub['DURATION_MS'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['ENERGY_categorical'] = pd.cut(sub['ENERGY'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['INSTRUMENTALNESS_categorical'] = pd.cut(sub['INSTRUMENTALNESS'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['SPEECHINESS_categorical'] = pd.cut(sub['SPEECHINESS'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['TEMPO_categorical'] = pd.cut(sub['TEMPO'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['VALENCE_categorical'] = pd.cut(sub['VALENCE'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['ARTIST_RELEASE_RATE_SINCE_TRACK_IN_QUESTION_RELEASE_categorical'] = pd.cut(sub['ARTIST_RELEASE_RATE_SINCE_TRACK_IN_QUESTION_RELEASE'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['MEDIAN_STREAMS_categorical'] = pd.cut(sub['MEDIAN_STREAMS'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['LINEAR_GRADIENT_categorical'] = pd.cut(sub['LINEAR_GRADIENT'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)\n",
    "sub['TEST_DATA_LINEAR_GRADIENT_categorical'] = pd.cut(sub['TEST_DATA_LINEAR_GRADIENT'], bins=n_bins, labels=['low', 'medium', 'high']).astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38c0e3e0-aa64-4747-9abf-53b27634d702",
   "metadata": {},
   "source": [
    "# Manual Buckets (sum of forecast errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a43b3f51-abae-4b16-b38c-d33b957e3660",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     4739.000000\n",
       "mean       148.624000\n",
       "std       1236.445707\n",
       "min          0.004000\n",
       "25%         17.838500\n",
       "50%         42.545000\n",
       "75%         97.788500\n",
       "max      65514.783000\n",
       "Name: SUM_FORECAST_ERRORS, dtype: float64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['SUM_FORECAST_ERRORS'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "c0b3f2d5-6ff0-462b-8026-88557f40e12a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = sub['SUM_FORECAST_ERRORS'].describe()[1]\n",
    "perc_25 = sub['SUM_FORECAST_ERRORS'].describe()[4]\n",
    "median = sub['SUM_FORECAST_ERRORS'].describe()[5]\n",
    "perc_75 = sub['SUM_FORECAST_ERRORS'].describe()[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4b7a73b3-ad86-4aa3-9ad3-7adab3f90309",
   "metadata": {},
   "outputs": [],
   "source": [
    "conditions = [(sub['SUM_FORECAST_ERRORS']<= perc_75),\n",
    "             ((sub['SUM_FORECAST_ERRORS']> perc_75) & (sub['SUM_FORECAST_ERRORS']<= mean)),\n",
    "             (sub['SUM_FORECAST_ERRORS'] > mean)]\n",
    "\n",
    "choices = ['doing_unnoticeably_better',\n",
    "          'doing_marginally_better',\n",
    "          'doing_much_better']\n",
    "\n",
    "sub['Performance_test_period']  = np.select(conditions, choices)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "749a9beb-7e3b-414b-aaa3-6de8eede94d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "doing_unnoticeably_better    3554\n",
       "doing_much_better             748\n",
       "doing_marginally_better       437\n",
       "Name: Performance_test_period, dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['Performance_test_period'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "5d11e77c-6802-483e-9c18-8ff342721f5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    4739.000000\n",
       "mean        1.549242\n",
       "std       136.270580\n",
       "min      -394.571429\n",
       "25%        -2.142857\n",
       "50%        -0.142857\n",
       "75%         1.714286\n",
       "max      9314.142857\n",
       "Name: TEST_DATA_LINEAR_GRADIENT, dtype: float64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['TEST_DATA_LINEAR_GRADIENT'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "c4d97bea-7336-4528-ab40-e5e0434d653b",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = sub['TEST_DATA_LINEAR_GRADIENT'].describe()[1]\n",
    "perc_25 = sub['TEST_DATA_LINEAR_GRADIENT'].describe()[4]\n",
    "median = sub['TEST_DATA_LINEAR_GRADIENT'].describe()[5]\n",
    "perc_75 = sub['TEST_DATA_LINEAR_GRADIENT'].describe()[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "aa78941c-ea77-4449-b032-d420eccb6de4",
   "metadata": {},
   "outputs": [],
   "source": [
    "conditions = [(sub['TEST_DATA_LINEAR_GRADIENT']<= perc_25),\n",
    "             ((sub['TEST_DATA_LINEAR_GRADIENT']> perc_25) & (sub['TEST_DATA_LINEAR_GRADIENT']<= median)),\n",
    "             ((sub['TEST_DATA_LINEAR_GRADIENT']> median) & (sub['TEST_DATA_LINEAR_GRADIENT']<= perc_75)),\n",
    "              (sub['TEST_DATA_LINEAR_GRADIENT'] > perc_75)]\n",
    "\n",
    "choices = ['down_linear_trend',\n",
    "          'marginally_down_linear_trend',\n",
    "          'marginally_up_linear_trend',\n",
    "          'up_linear_trend']\n",
    "\n",
    "sub['Slope_test_period']  = np.select(conditions, choices)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "611349df-49a6-48fd-a977-6d242faf1eb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "marginally_down_linear_trend    1221\n",
       "down_linear_trend               1206\n",
       "up_linear_trend                 1171\n",
       "marginally_up_linear_trend      1141\n",
       "Name: Slope_test_period, dtype: int64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['Slope_test_period'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "c0b36427-fec8-4b0c-a1a4-023506f055c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Performance_test_period</th>\n",
       "      <th>Slope_test_period</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>marginally_down_linear_trend</td>\n",
       "      <td>1086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>marginally_up_linear_trend</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>down_linear_trend</td>\n",
       "      <td>792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>up_linear_trend</td>\n",
       "      <td>677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>doing_much_better</td>\n",
       "      <td>up_linear_trend</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>doing_much_better</td>\n",
       "      <td>down_linear_trend</td>\n",
       "      <td>289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>doing_marginally_better</td>\n",
       "      <td>up_linear_trend</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>doing_marginally_better</td>\n",
       "      <td>down_linear_trend</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>doing_marginally_better</td>\n",
       "      <td>marginally_up_linear_trend</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>doing_marginally_better</td>\n",
       "      <td>marginally_down_linear_trend</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>doing_much_better</td>\n",
       "      <td>marginally_down_linear_trend</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>doing_much_better</td>\n",
       "      <td>marginally_up_linear_trend</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Performance_test_period             Slope_test_period  count\n",
       "0   doing_unnoticeably_better  marginally_down_linear_trend   1086\n",
       "1   doing_unnoticeably_better    marginally_up_linear_trend    999\n",
       "2   doing_unnoticeably_better             down_linear_trend    792\n",
       "3   doing_unnoticeably_better               up_linear_trend    677\n",
       "4           doing_much_better               up_linear_trend    338\n",
       "5           doing_much_better             down_linear_trend    289\n",
       "6     doing_marginally_better               up_linear_trend    156\n",
       "7     doing_marginally_better             down_linear_trend    125\n",
       "8     doing_marginally_better    marginally_up_linear_trend     86\n",
       "9     doing_marginally_better  marginally_down_linear_trend     70\n",
       "10          doing_much_better  marginally_down_linear_trend     65\n",
       "11          doing_much_better    marginally_up_linear_trend     56"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_combos = sub[['Performance_test_period', 'Slope_test_period']].value_counts().reset_index(name='count')\n",
    "unique_combos\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "5ee610bb-b077-475e-8107-7aa8bdcd5ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_to_save = sub[['ISRC','TRANSACTION_COUNTRY_CODE','AVERAGE_DAILY_STREAMS','SUM_FORECAST_ERRORS', 'Performance_test_period', 'Slope_test_period']].copy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "46d2c10c-82d1-4fcb-9b8b-92e7c7852764",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>TRANSACTION_COUNTRY_CODE</th>\n",
       "      <th>AVERAGE_DAILY_STREAMS</th>\n",
       "      <th>SUM_FORECAST_ERRORS</th>\n",
       "      <th>Performance_test_period</th>\n",
       "      <th>Slope_test_period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>QMBZ91955892</td>\n",
       "      <td>GB</td>\n",
       "      <td>66.105263</td>\n",
       "      <td>41.462</td>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>marginally_up_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>USA560434628</td>\n",
       "      <td>GB</td>\n",
       "      <td>65.353383</td>\n",
       "      <td>67.308</td>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>up_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>USFW40671102</td>\n",
       "      <td>GB</td>\n",
       "      <td>65.428571</td>\n",
       "      <td>104.222</td>\n",
       "      <td>doing_marginally_better</td>\n",
       "      <td>down_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>QZMHK2059350</td>\n",
       "      <td>GB</td>\n",
       "      <td>66.112782</td>\n",
       "      <td>18.479</td>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>marginally_down_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>DEA451711960</td>\n",
       "      <td>GB</td>\n",
       "      <td>66.300752</td>\n",
       "      <td>45.861</td>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>marginally_up_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10876</th>\n",
       "      <td>QM6MZ2005307</td>\n",
       "      <td>GB</td>\n",
       "      <td>296.984962</td>\n",
       "      <td>266.475</td>\n",
       "      <td>doing_much_better</td>\n",
       "      <td>marginally_down_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10878</th>\n",
       "      <td>GBHAH1600218</td>\n",
       "      <td>GB</td>\n",
       "      <td>297.052632</td>\n",
       "      <td>8.693</td>\n",
       "      <td>doing_unnoticeably_better</td>\n",
       "      <td>marginally_down_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10881</th>\n",
       "      <td>USSTT2000187</td>\n",
       "      <td>GB</td>\n",
       "      <td>295.195489</td>\n",
       "      <td>386.763</td>\n",
       "      <td>doing_much_better</td>\n",
       "      <td>down_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10887</th>\n",
       "      <td>GBJXP2143701</td>\n",
       "      <td>GB</td>\n",
       "      <td>290.105263</td>\n",
       "      <td>1379.371</td>\n",
       "      <td>doing_much_better</td>\n",
       "      <td>down_linear_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10891</th>\n",
       "      <td>GBBQY9902007</td>\n",
       "      <td>GB</td>\n",
       "      <td>298.954887</td>\n",
       "      <td>123.812</td>\n",
       "      <td>doing_marginally_better</td>\n",
       "      <td>down_linear_trend</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4739 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               ISRC TRANSACTION_COUNTRY_CODE  AVERAGE_DAILY_STREAMS  \\\n",
       "1      QMBZ91955892                       GB              66.105263   \n",
       "5      USA560434628                       GB              65.353383   \n",
       "7      USFW40671102                       GB              65.428571   \n",
       "8      QZMHK2059350                       GB              66.112782   \n",
       "10     DEA451711960                       GB              66.300752   \n",
       "...             ...                      ...                    ...   \n",
       "10876  QM6MZ2005307                       GB             296.984962   \n",
       "10878  GBHAH1600218                       GB             297.052632   \n",
       "10881  USSTT2000187                       GB             295.195489   \n",
       "10887  GBJXP2143701                       GB             290.105263   \n",
       "10891  GBBQY9902007                       GB             298.954887   \n",
       "\n",
       "       SUM_FORECAST_ERRORS    Performance_test_period  \\\n",
       "1                   41.462  doing_unnoticeably_better   \n",
       "5                   67.308  doing_unnoticeably_better   \n",
       "7                  104.222    doing_marginally_better   \n",
       "8                   18.479  doing_unnoticeably_better   \n",
       "10                  45.861  doing_unnoticeably_better   \n",
       "...                    ...                        ...   \n",
       "10876              266.475          doing_much_better   \n",
       "10878                8.693  doing_unnoticeably_better   \n",
       "10881              386.763          doing_much_better   \n",
       "10887             1379.371          doing_much_better   \n",
       "10891              123.812    doing_marginally_better   \n",
       "\n",
       "                  Slope_test_period  \n",
       "1        marginally_up_linear_trend  \n",
       "5                   up_linear_trend  \n",
       "7                 down_linear_trend  \n",
       "8      marginally_down_linear_trend  \n",
       "10       marginally_up_linear_trend  \n",
       "...                             ...  \n",
       "10876  marginally_down_linear_trend  \n",
       "10878  marginally_down_linear_trend  \n",
       "10881             down_linear_trend  \n",
       "10887             down_linear_trend  \n",
       "10891             down_linear_trend  \n",
       "\n",
       "[4739 rows x 6 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_to_save"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}