{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d50df237", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/core/computation/expressions.py:21: UserWarning: Pandas requires version '2.8.0' or newer of 'numexpr' (version '2.7.3' currently installed).\n", " from pandas.core.computation.check import NUMEXPR_INSTALLED\n" ] } ], "source": [ "import pandas as pd\n", "import os\n", "import boto3 \n", "from datetime import datetime, date, timedelta" ] }, { "cell_type": "code", "execution_count": 2, "id": "b8268473", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/fsspec/registry.py:272: UserWarning: Your installed version of s3fs is very old and known to cause\n", "severe performance issues, see also https://github.com/dask/dask/issues/10276\n", "\n", "To fix, you should specify a lower version bound on s3fs, or\n", "update the current installation.\n", "\n", " warnings.warn(s3_msg)\n" ] }, { "data": { "text/plain": [ "(6036100, 6)" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fourier = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/FOURIERTABLE_CHUNKS_01678_20231110-122556.csv')\n", "fourier.shape" ] }, { "cell_type": "code", "execution_count": 3, "id": "c2b48451", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 6036100 entries, 0 to 6036099\n", "Data columns (total 6 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 ACTIVITY_DATE object \n", " 1 Fourier object \n", " 2 Fourier_real_part float64\n", " 3 ISRC object \n", " 4 Inflection_Point int64 \n", " 5 STREAMS int64 \n", "dtypes: float64(1), int64(2), object(3)\n", "memory usage: 276.3+ MB\n" ] } ], "source": [ "fourier.info()" ] }, { "cell_type": "code", "execution_count": 4, "id": "2e11d289", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ACTIVITY_DATEFourierFourier_real_partISRCInflection_PointSTREAMS
02023-06-04(19.98039739025349+0j)19.980397CAPB70990202030
12023-06-05(20.2105110657345+0j)20.210511CAPB70990202018
22023-06-06(20.441765804356603-4.0602442043434295e-16j)20.441766CAPB70990202022
32023-06-07(20.673066478326433-1.0150610510858574e-16j)20.673066CAPB70990202129
42023-06-08(20.90328045354309-2.0301221021717148e-16j)20.903280CAPB70990202021
\n", "
" ], "text/plain": [ " ACTIVITY_DATE Fourier \\\n", "0 2023-06-04 (19.98039739025349+0j) \n", "1 2023-06-05 (20.2105110657345+0j) \n", "2 2023-06-06 (20.441765804356603-4.0602442043434295e-16j) \n", "3 2023-06-07 (20.673066478326433-1.0150610510858574e-16j) \n", "4 2023-06-08 (20.90328045354309-2.0301221021717148e-16j) \n", "\n", " Fourier_real_part ISRC Inflection_Point STREAMS \n", "0 19.980397 CAPB70990202 0 30 \n", "1 20.210511 CAPB70990202 0 18 \n", "2 20.441766 CAPB70990202 0 22 \n", "3 20.673066 CAPB70990202 1 29 \n", "4 20.903280 CAPB70990202 0 21 " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fourier.head()" ] }, { "cell_type": "code", "execution_count": 4, "id": "d32befe9", "metadata": {}, "outputs": [], "source": [ "\n", "# fourier['ACTIVITY_DATE'] = pd.to_datetime(fourier['ACTIVITY_DATE'])\n", "# fourier['ACTIVITY_DATE'] = fourier['ACTIVITY_DATE'].dt.date\n", "\n", "# first_day_pred = fourier['ACTIVITY_DATE'].max() - timedelta(days=7)\n", "# list_for_pred = fourier[(fourier['Inflection_Point']==1) & \n", "# (fourier['ACTIVITY_DATE'] > first_day_pred)]['ISRC'].unique().tolist()\n", "# f2 = fourier[fourier['ISRC'].isin(list_for_pred)].copy()\n", "\n", "\n", "\n", "# f2.shape" ] }, { "cell_type": "code", "execution_count": 5, "id": "fe45384a", "metadata": {}, "outputs": [], "source": [ "#f2['ISRC'].nunique()" ] }, { "cell_type": "code", "execution_count": 6, "id": "8378cdb1", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(8128, 11)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "arima = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/ARIMA_chunks/ARIMA_CHUNKS_0178_20231110-123152.csv')\n", "arima.shape" ] }, { "cell_type": "code", "execution_count": 7, "id": "32a667cc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 8128 entries, 0 to 8127\n", "Data columns (total 11 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 ISRC 8128 non-null object \n", " 1 pred_type 8128 non-null object \n", " 2 mse 8128 non-null float64\n", " 3 avg_streams_train 8128 non-null float64\n", " 4 avg_streams_test 8128 non-null float64\n", " 5 median_streams_train 8128 non-null float64\n", " 6 median_streams_test 8128 non-null float64\n", " 7 linear_gradient_train 8128 non-null float64\n", " 8 linear_gradient_test 8128 non-null float64\n", " 9 sum_forecast_errors 8128 non-null float64\n", " 10 len_df 8128 non-null int64 \n", "dtypes: float64(8), int64(1), object(2)\n", "memory usage: 698.6+ KB\n" ] } ], "source": [ "arima.info()" ] }, { "cell_type": "code", "execution_count": null, "id": "397c8a74", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 8, "id": "949a8142", "metadata": {}, "outputs": [], "source": [ "#fourier[fourier['ISRC']=='SEVOZ2000304']" ] }, { "cell_type": "code", "execution_count": 9, "id": "ee3b8f5e", "metadata": {}, "outputs": [], "source": [ "\n", "df = pd.read_csv('s3://dev-cucumbers/eimpara/Fourier/2023_data/data_for_2023_analysis_20231107-112155.csv')\n", "arima = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/ARIMA_chunks/ARIMA_CHUNKS_0178_20231110-123152.csv')\n", "\n", "df['ACTIVITY_DATE'] = pd.to_datetime(df['ACTIVITY_DATE'])\n", "df['ACTIVITY_DATE'] = df['ACTIVITY_DATE'].dt.date\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "916199d5", "metadata": {}, "outputs": [], "source": [ "def load_full_data(df):\n", " list_for_pred = arima['ISRC'].unique().tolist()\n", " subset_for_pred = df[df['ISRC'].isin(list_for_pred)].copy()\n", " return subset_for_pred\n", "\n", "df = load_full_data(df)" ] }, { "cell_type": "code", "execution_count": 11, "id": "af843704", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ISRCACTIVITY_DATETRACK_NAMERELEASE_DATEARTIST_NAMEARTIST_IDSTREAMS
0US33Q04016032023-09-21Nobody Knows Me2004-07-17Aya40077021
1USDO198000212023-09-21Don Quixote1998-10-06Luis Villegas42315836
30DEU2416029112023-09-21So Are You2016-06-17Dub Fx819030211
51CHB7018001882023-09-21Breathe 2.02018-11-30Vincent275449025
52CHB7017001132023-09-21Handful of Gold2017-07-21CAZZETTE289443880
\n", "
" ], "text/plain": [ " ISRC ACTIVITY_DATE TRACK_NAME RELEASE_DATE ARTIST_NAME \\\n", "0 US33Q0401603 2023-09-21 Nobody Knows Me 2004-07-17 Aya \n", "1 USDO19800021 2023-09-21 Don Quixote 1998-10-06 Luis Villegas \n", "30 DEU241602911 2023-09-21 So Are You 2016-06-17 Dub Fx \n", "51 CHB701800188 2023-09-21 Breathe 2.0 2018-11-30 Vincent \n", "52 CHB701700113 2023-09-21 Handful of Gold 2017-07-21 CAZZETTE \n", "\n", " ARTIST_ID STREAMS \n", "0 400770 21 \n", "1 423158 36 \n", "30 819030 211 \n", "51 2754490 25 \n", "52 2894438 80 " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 12, "id": "7867effc", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ISRCACTIVITY_DATETRACK_NAMERELEASE_DATEARTIST_NAMEARTIST_IDSTREAMS
20887SEVOZ20003042023-10-06Fly Me to the Moon2021-02-26Isabella Lundgren70195720
48368SEVOZ20003042023-10-09Fly Me to the Moon2021-02-26Isabella Lundgren70195719
59487SEVOZ20003042023-10-02Fly Me to the Moon2021-02-26Isabella Lundgren70195711
65273SEVOZ20003042023-10-08Fly Me to the Moon2021-02-26Isabella Lundgren70195720
188491SEVOZ20003042023-09-08Fly Me to the Moon2021-02-26Isabella Lundgren70195721
........................
11616916SEVOZ20003042023-06-10Fly Me to the Moon2021-02-26Isabella Lundgren70195730
11697758SEVOZ20003042023-10-16Fly Me to the Moon2021-02-26Isabella Lundgren70195718
11861713SEVOZ20003042023-07-07Fly Me to the Moon2021-02-26Isabella Lundgren70195726
11931799SEVOZ20003042023-08-12Fly Me to the Moon2021-02-26Isabella Lundgren70195727
12072568SEVOZ20003042023-09-19Fly Me to the Moon2021-02-26Isabella Lundgren7019579
\n", "

140 rows × 7 columns

\n", "
" ], "text/plain": [ " ISRC ACTIVITY_DATE TRACK_NAME RELEASE_DATE \\\n", "20887 SEVOZ2000304 2023-10-06 Fly Me to the Moon 2021-02-26 \n", "48368 SEVOZ2000304 2023-10-09 Fly Me to the Moon 2021-02-26 \n", "59487 SEVOZ2000304 2023-10-02 Fly Me to the Moon 2021-02-26 \n", "65273 SEVOZ2000304 2023-10-08 Fly Me to the Moon 2021-02-26 \n", "188491 SEVOZ2000304 2023-09-08 Fly Me to the Moon 2021-02-26 \n", "... ... ... ... ... \n", "11616916 SEVOZ2000304 2023-06-10 Fly Me to the Moon 2021-02-26 \n", "11697758 SEVOZ2000304 2023-10-16 Fly Me to the Moon 2021-02-26 \n", "11861713 SEVOZ2000304 2023-07-07 Fly Me to the Moon 2021-02-26 \n", "11931799 SEVOZ2000304 2023-08-12 Fly Me to the Moon 2021-02-26 \n", "12072568 SEVOZ2000304 2023-09-19 Fly Me to the Moon 2021-02-26 \n", "\n", " ARTIST_NAME ARTIST_ID STREAMS \n", "20887 Isabella Lundgren 701957 20 \n", "48368 Isabella Lundgren 701957 19 \n", "59487 Isabella Lundgren 701957 11 \n", "65273 Isabella Lundgren 701957 20 \n", "188491 Isabella Lundgren 701957 21 \n", "... ... ... ... \n", "11616916 Isabella Lundgren 701957 30 \n", "11697758 Isabella Lundgren 701957 18 \n", "11861713 Isabella Lundgren 701957 26 \n", "11931799 Isabella Lundgren 701957 27 \n", "12072568 Isabella Lundgren 701957 9 \n", "\n", "[140 rows x 7 columns]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[df['ISRC']=='SEVOZ2000304']" ] }, { "cell_type": "code", "execution_count": 2, "id": "faedb34b", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/fsspec/registry.py:272: UserWarning: Your installed version of s3fs is very old and known to cause\n", "severe performance issues, see also https://github.com/dask/dask/issues/10276\n", "\n", "To fix, you should specify a lower version bound on s3fs, or\n", "update the current installation.\n", "\n", " warnings.warn(s3_msg)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ISRCACTIVITY_DATESTREAMSPredictedlen_df
0US43C16142042023-06-041910.826566140
1US43C16142042023-06-051513.536349140
2US43C16142042023-06-061515.321240140
3US43C16142042023-06-071916.268454140
4US43C16142042023-06-081817.695292140
\n", "
" ], "text/plain": [ " ISRC ACTIVITY_DATE STREAMS Predicted len_df\n", "0 US43C1614204 2023-06-04 19 10.826566 140\n", "1 US43C1614204 2023-06-05 15 13.536349 140\n", "2 US43C1614204 2023-06-06 15 15.321240 140\n", "3 US43C1614204 2023-06-07 19 16.268454 140\n", "4 US43C1614204 2023-06-08 18 17.695292 140" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test1 = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/ARIMA_chunks/Prediction_chunks/1_1699641657479874.8_20231110-184216.csv')\n", "test1.head()\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "a81a1a44", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ISRCACTIVITY_DATESTREAMSPredictedlen_df
0FR5U199001072023-06-041216.089729140
1FR5U199001072023-06-051015.439914140
2FR5U199001072023-06-061511.072805140
3FR5U199001072023-06-072014.931367140
4FR5U199001072023-06-081514.205376140
\n", "
" ], "text/plain": [ " ISRC ACTIVITY_DATE STREAMS Predicted len_df\n", "0 FR5U19900107 2023-06-04 12 16.089729 140\n", "1 FR5U19900107 2023-06-05 10 15.439914 140\n", "2 FR5U19900107 2023-06-06 15 11.072805 140\n", "3 FR5U19900107 2023-06-07 20 14.931367 140\n", "4 FR5U19900107 2023-06-08 15 14.205376 140" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test2 = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/ARIMA_chunks/Prediction_chunks/0_1699641657479874.8_20231110-184259.csv')\n", "test2.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "46443846", "metadata": {}, "outputs": [], "source": [ "fourier = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/FOURIERTABLE_CHUNKS_01678_20231110-122556.csv')\n", "fourier.shape" ] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" } }, "nbformat": 4, "nbformat_minor": 5 }