{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "9b4de28d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/core/computation/expressions.py:21: UserWarning: Pandas requires version '2.8.0' or newer of 'numexpr' (version '2.7.3' currently installed).\n", " from pandas.core.computation.check import NUMEXPR_INSTALLED\n" ] } ], "source": [ "!pip -q install snowflake-connector-python pytest pytest-sugar \n", "!pip -q install pyecharts absl-py\n", "!pip -q install statsmodels\n", "!pip -q install pmdarima\n", "import snowflake.connector" ] }, { "cell_type": "code", "execution_count": 2, "id": "5e6da25e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Matplotlib is building the font cache; this may take a moment.\n" ] } ], "source": [ "# main imports\n", "import snowflake.connector\n", "import pandas as pd\n", "import pyecharts as echarts\n", "import os\n", "import boto3 \n", "\n", "import math\n", "import random\n", "import scipy\n", "import numpy as np\n", "import statsmodels.formula.api as smf\n", "import statsmodels.api as sm\n", "import pmdarima as pm \n", "import time\n", "\n", "from datetime import datetime, date, timedelta\n", "from statsmodels.tsa.statespace.sarimax import SARIMAX\n", "from statsmodels.tsa.arima.model import ARIMA\n", "from statsmodels.graphics.tsaplots import plot_acf, plot_pacf\n", "from statsmodels.tsa.seasonal import seasonal_decompose\n", "from statsmodels.tools.eval_measures import mse,rmse, meanabs\n", "from statsmodels.tsa.stattools import adfuller\n", "from statsmodels.tsa.statespace.tools import diff\n", "from scipy import fftpack" ] }, { "cell_type": "code", "execution_count": 3, "id": "9ca05bb7", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/fsspec/registry.py:272: UserWarning: Your installed version of s3fs is very old and known to cause\n", "severe performance issues, see also https://github.com/dask/dask/issues/10276\n", "\n", "To fix, you should specify a lower version bound on s3fs, or\n", "update the current installation.\n", "\n", " warnings.warn(s3_msg)\n" ] } ], "source": [ "df = pd.read_csv('s3://dev-cucumbers/eimpara/Fourier/2023_data/data_for_2023_analysis_20231107-112155.csv')\n", "#arima = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/ARIMA_chunks/ARIMA_CHUNKS_0178_20231110-123152.csv')" ] }, { "cell_type": "code", "execution_count": 4, "id": "13dcb7ab", "metadata": {}, "outputs": [], "source": [ "subset_for_pred = df[df['ISRC']=='GBBLG0001891']" ] }, { "cell_type": "code", "execution_count": 5, "id": "d1694b63", "metadata": {}, "outputs": [], "source": [ "onedf = subset_for_pred[['ACTIVITY_DATE', 'STREAMS']].copy()\n", "onedf = onedf.sort_values(by=['ACTIVITY_DATE'])\n", "onedf['ACTIVITY_DATE'] = pd.to_datetime(onedf['ACTIVITY_DATE'])\n", "(train, test) = (onedf.iloc[:133], onedf.iloc[133:])" ] }, { "cell_type": "code", "execution_count": 7, "id": "b451cd8b", "metadata": {}, "outputs": [], "source": [ "onedf.index = pd.to_datetime(onedf['ACTIVITY_DATE'])\n", "train.index = pd.to_datetime(train['ACTIVITY_DATE'])\n", "test.index = pd.to_datetime(test['ACTIVITY_DATE'])\n", "auto_df = train[['STREAMS']].copy()" ] }, { "cell_type": "code", "execution_count": 8, "id": "80792c02", "metadata": {}, "outputs": [], "source": [ "auto_model = pm.auto_arima(auto_df, start_p=1, start_q=1,\n", " test='adf',\n", " max_p=3, max_q=3, m=7,\n", " start_P=0, seasonal=True,\n", " d=None, D=1, trace=False,\n", " error_action='ignore', #maxiter= 3000000,\n", " suppress_warnings=True, #stationary=False,\n", " stepwise=True)" ] }, { "cell_type": "code", "execution_count": 9, "id": "7f195201", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/statsmodels/tsa/base/tsa_model.py:473: ValueWarning: No frequency information was provided, so inferred frequency D will be used.\n", " self._init_dates(dates, freq)\n", "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/statsmodels/tsa/base/tsa_model.py:473: ValueWarning: No frequency information was provided, so inferred frequency D will be used.\n", " self._init_dates(dates, freq)\n", "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/statsmodels/tsa/base/tsa_model.py:473: ValueWarning: No frequency information was provided, so inferred frequency D will be used.\n", " self._init_dates(dates, freq)\n" ] } ], "source": [ "arima_model = ARIMA(train['STREAMS'], order=auto_model.order, seasonal_order=auto_model.seasonal_order,enforce_stationarity=False).fit()" ] }, { "cell_type": "code", "execution_count": 19, "id": "5770d379", "metadata": {}, "outputs": [], "source": [ "def iterate_group_by_key(df, col_for_key, sort_data=True):\n", " if sort_data:\n", " df = df.sort_values(by=col_for_key)\n", " def key_at(i): return np.array(df[col_for_key].iloc[i])\n", " index, size = (0, df.shape[0])\n", " while index < size:\n", " current_key = key_at(index)\n", " res = []\n", " while index < size and list(current_key) == list(key_at(index)):\n", " res.append(df.iloc[index])\n", " index = index + 1\n", " resdf = pd.DataFrame(res, columns=df.columns)\n", " yield resdf\n", " \n", "def auto_arima(df_isrc):\n", "\n", " cutoff_train_test_sets = 133\n", " (train, test) = (df_isrc.iloc[:cutoff_train_test_sets], df_isrc.iloc[cutoff_train_test_sets:])\n", " train.index = pd.to_datetime(train['ACTIVITY_DATE'])\n", " train = train.sort_index(axis = 0)\n", " auto_df = train[['STREAMS']].copy()\n", " auto_model = pm.auto_arima(auto_df, start_p=1, start_q=1,\n", " test='adf',\n", " max_p=3, max_q=3, m=7,\n", " start_P=0, seasonal=True,\n", " d=None, D=1, trace=False,\n", " error_action='ignore', \n", " suppress_warnings=True, #stationary=False,\n", " stepwise=True)\n", " return (auto_model.seasonal_order, auto_model.order)\n", "\n", "\n", "def arima_iscrs(df):\n", "\n", " new_df = pd.DataFrame(columns=['ISRC', 'ACTIVITY_DATE', 'STREAMS', 'Predicted','len_df'])\n", " count = 1\n", " for subdf in iterate_group_by_key(df, ['ISRC']):\n", " df_isrc = subdf\n", " isrc = df_isrc['ISRC'].iloc[0]\n", " df_isrc = df_isrc.sort_values(by=['ACTIVITY_DATE'], ascending=True)\n", " df_isrc = df_isrc.reset_index()\n", " activity = df_isrc['ACTIVITY_DATE']\n", " streams = df_isrc['STREAMS']\n", " \n", " try:\n", " cutoff_train_test_sets = 133\n", " start_pred = 133 \n", " end_pred = 139\n", " (train, test) = (df_isrc.iloc[:cutoff_train_test_sets], df_isrc.iloc[cutoff_train_test_sets:])\n", " (end_test, end_train) = (len(test), len(train))\n", " (seasonal_order, order) = auto_arima(train)\n", " arima_model = ARIMA(train['STREAMS'], order=order, seasonal_order=seasonal_order,enforce_stationarity=False).fit()\n", " pred = arima_model.get_prediction(start= 1, end = (end_train + end_test-1), dynamic=False)\n", " predicted_values = pred.predicted_mean[:end_pred]\n", " isrcs_col = np.full(\n", " shape=df_isrc.shape[0],\n", " fill_value=isrc,\n", " dtype=object\n", " )\n", " len_df_col = np.full(\n", " shape=df_isrc.shape[0],\n", " fill_value=df_isrc.shape[0],\n", " dtype=int\n", " )\n", " new_df = pd.concat([new_df, pd.DataFrame({\n", " 'ISRC': isrcs_col, \n", " 'ACTIVITY_DATE': activity,\n", " 'STREAMS': streams,\n", " 'Predicted': predicted_values,\n", " 'len_df': len_df_col\n", " }, columns=new_df.columns)])\n", " if count % 10 ==0:\n", " print(\"Worker processed {} ISRCs.\".format(count))\n", " count = count+1\n", " except Exception as e:\n", " print(\"Error while processing ISRC {}: '{}'\".format(isrc, e))\n", " return new_df" ] }, { "cell_type": "code", "execution_count": 17, "id": "2565b94a", "metadata": {}, "outputs": [], "source": [ "(seasonal_order, order) = auto_arima(onedf)" ] }, { "cell_type": "code", "execution_count": 21, "id": "d27842ce", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_16455/3344622057.py:65: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n", " new_df = pd.concat([new_df, pd.DataFrame({\n" ] } ], "source": [ "test = arima_iscrs(subset_for_pred)" ] }, { "cell_type": "code", "execution_count": 22, "id": "4c9286d8", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ISRCACTIVITY_DATESTREAMSPredictedlen_df
0GBBLG00018912023-06-0453NaN140
1GBBLG00018912023-06-055723.305864140
2GBBLG00018912023-06-065937.713351140
3GBBLG00018912023-06-074744.245136140
4GBBLG00018912023-06-084740.349078140
..................
135GBBLG00018912023-10-175666.970418140
136GBBLG00018912023-10-185662.505630140
137GBBLG00018912023-10-195665.912332140
138GBBLG00018912023-10-204968.185283140
139GBBLG00018912023-10-214765.009099140
\n", "

140 rows × 5 columns

\n", "
" ], "text/plain": [ " ISRC ACTIVITY_DATE STREAMS Predicted len_df\n", "0 GBBLG0001891 2023-06-04 53 NaN 140\n", "1 GBBLG0001891 2023-06-05 57 23.305864 140\n", "2 GBBLG0001891 2023-06-06 59 37.713351 140\n", "3 GBBLG0001891 2023-06-07 47 44.245136 140\n", "4 GBBLG0001891 2023-06-08 47 40.349078 140\n", ".. ... ... ... ... ...\n", "135 GBBLG0001891 2023-10-17 56 66.970418 140\n", "136 GBBLG0001891 2023-10-18 56 62.505630 140\n", "137 GBBLG0001891 2023-10-19 56 65.912332 140\n", "138 GBBLG0001891 2023-10-20 49 68.185283 140\n", "139 GBBLG0001891 2023-10-21 47 65.009099 140\n", "\n", "[140 rows x 5 columns]" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test" ] }, { "cell_type": "code", "execution_count": null, "id": "d4133c47", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" } }, "nbformat": 4, "nbformat_minor": 5 }