{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "b5eeafcb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 1, "id": "4920d5a4", "metadata": {}, "outputs": [], "source": [ "!pip -q install pyecharts absl-py\n", "!pip -q install statsmodels\n", "!pip -q install pmdarima" ] }, { "cell_type": "code", "execution_count": 2, "id": "a5b766fc", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/pandas/core/computation/expressions.py:21: UserWarning: Pandas requires version '2.8.0' or newer of 'numexpr' (version '2.7.3' currently installed).\n", " from pandas.core.computation.check import NUMEXPR_INSTALLED\n" ] } ], "source": [ "import pandas as pd\n", "import pyecharts as echarts\n", "import os\n", "import boto3 \n", "\n", "import math\n", "import random\n", "import scipy\n", "import numpy as np\n", "import statsmodels.formula.api as smf\n", "import statsmodels.api as sm\n", "import pmdarima as pm \n", "import time\n", "\n", "from datetime import datetime, date, timedelta\n", "from statsmodels.tsa.statespace.sarimax import SARIMAX\n", "from statsmodels.tsa.arima.model import ARIMA\n", "from statsmodels.graphics.tsaplots import plot_acf, plot_pacf\n", "from statsmodels.tsa.seasonal import seasonal_decompose\n", "from statsmodels.tools.eval_measures import mse,rmse, meanabs\n", "from statsmodels.tsa.stattools import adfuller\n", "from statsmodels.tsa.statespace.tools import diff\n", "from scipy import fftpack\n", "from multiprocess import Pool" ] }, { "cell_type": "code", "execution_count": 3, "id": "937ebc9e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/fsspec/registry.py:272: UserWarning: Your installed version of s3fs is very old and known to cause\n", "severe performance issues, see also https://github.com/dask/dask/issues/10276\n", "\n", "To fix, you should specify a lower version bound on s3fs, or\n", "update the current installation.\n", "\n", " warnings.warn(s3_msg)\n" ] }, { "data": { "text/plain": [ "(1207220, 6)" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "original = pd.read_csv('s3://dev-cucumbers/eimpara/Moments_2023_batches/Fourier_table_NEW_DATA_0_20231108-111429.csv')\n", "original.shape" ] }, { "cell_type": "code", "execution_count": 4, "id": "e9d45cfe", "metadata": {}, "outputs": [], "source": [ "df = original.copy()" ] }, { "cell_type": "code", "execution_count": 5, "id": "cb664186", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Min activity date: 2023-06-04\n", "Max activity date: 2023-10-21\n" ] } ], "source": [ "df['ACTIVITY_DATE'] = pd.to_datetime(df['ACTIVITY_DATE'])\n", "df['ACTIVITY_DATE'] = df['ACTIVITY_DATE'].dt.date\n", "\n", "print('Min activity date: ', df['ACTIVITY_DATE'].min())\n", "print('Max activity date: ', df['ACTIVITY_DATE'].max())" ] }, { "cell_type": "code", "execution_count": 6, "id": "cf068e53", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ACTIVITY_DATEFourierFourier_real_partISRCInflection_PointSTREAMS
02023-06-04(19.98039739025349+0j)19.980397CAPB70990202030
12023-06-05(20.2105110657345+0j)20.210511CAPB70990202018
22023-06-06(20.441765804356603-4.0602442043434295e-16j)20.441766CAPB70990202022
32023-06-07(20.673066478326433-1.0150610510858574e-16j)20.673066CAPB70990202129
42023-06-08(20.90328045354309-2.0301221021717148e-16j)20.903280CAPB70990202021
\n", "
" ], "text/plain": [ " ACTIVITY_DATE Fourier \\\n", "0 2023-06-04 (19.98039739025349+0j) \n", "1 2023-06-05 (20.2105110657345+0j) \n", "2 2023-06-06 (20.441765804356603-4.0602442043434295e-16j) \n", "3 2023-06-07 (20.673066478326433-1.0150610510858574e-16j) \n", "4 2023-06-08 (20.90328045354309-2.0301221021717148e-16j) \n", "\n", " Fourier_real_part ISRC Inflection_Point STREAMS \n", "0 19.980397 CAPB70990202 0 30 \n", "1 20.210511 CAPB70990202 0 18 \n", "2 20.441766 CAPB70990202 0 22 \n", "3 20.673066 CAPB70990202 1 29 \n", "4 20.903280 CAPB70990202 0 21 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 7, "id": "697a5a26", "metadata": {}, "outputs": [], "source": [ "def load_full_data(df):\n", " first_day_pred = df['ACTIVITY_DATE'].max() - timedelta(days=7)\n", " list_for_pred = df[(df['Inflection_Point']==1) & \n", " (df['ACTIVITY_DATE'] > first_day_pred)]['ISRC'].unique().tolist()\n", " subset_for_pred = df[df['ISRC'].isin(list_for_pred)].copy()\n", " return subset_for_pred\n", "\n", "\n", "def save_dataframe_s3(df):\n", " \n", " '''Saving table to S3'''\n", " \n", " s3 = boto3.client('s3')\n", " bucket_name = 'dev-cucumbers'\n", " today = datetime.today().strftime('%Y%m%d-%H%M%S')\n", " filepath = \"eimpara/Moments_2023_batches/Fourier_TEST_{}.csv\".format(today)\n", " csv_buffer = df.to_csv(index=False).encode('utf-8')\n", " s3.put_object(Body=csv_buffer, Bucket=bucket_name, Key=filepath)\n", " print(f\"Table saved to S3 bucket: {bucket_name}, with file name: {filepath}\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "4ce74970", "metadata": {}, "outputs": [], "source": [ "test = load_full_data(df)" ] }, { "cell_type": "code", "execution_count": 9, "id": "15cfaa7a", "metadata": {}, "outputs": [], "source": [ "IDs = test['ISRC'].unique().tolist()\n", "sample_size = 100\n", "random.seed(2003)\n", "random_sample = random.sample(IDs, k=sample_size)\n", "subset = test[test['ISRC'].isin(random_sample)]" ] }, { "cell_type": "code", "execution_count": 10, "id": "b9229b8d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(14000, 6)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "subset.shape" ] }, { "cell_type": "code", "execution_count": 11, "id": "defb5b27", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "100" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "subset['ISRC'].nunique()" ] }, { "cell_type": "code", "execution_count": 12, "id": "d2250a89", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Table saved to S3 bucket: dev-cucumbers, with file name: eimpara/Moments_2023_batches/Fourier_TEST_20231117-134334.csv\n" ] } ], "source": [ "save_dataframe_s3(subset)" ] }, { "cell_type": "code", "execution_count": null, "id": "c27accb4", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" } }, "nbformat": 4, "nbformat_minor": 5 }