{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "85a16823",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Package(s) not found: snowflake-connector-python\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip show snowflake-connector-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4da587c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aiohttp @ file:///home/conda/feedstock_root/build_artifacts/aiohttp_1701099469104/work\r\n",
      "aiosignal @ file:///home/conda/feedstock_root/build_artifacts/aiosignal_1667935791922/work\r\n",
      "alabaster @ file:///home/conda/feedstock_root/build_artifacts/alabaster_1704848697227/work\r\n",
      "anyio @ file:///home/conda/feedstock_root/build_artifacts/anyio_1702909220329/work\r\n",
      "argon2-cffi @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi_1692818318753/work\r\n",
      "argon2-cffi-bindings @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi-bindings_1695386546427/work\r\n",
      "arrow @ file:///home/conda/feedstock_root/build_artifacts/arrow_1696128962909/work\r\n",
      "astroid @ file:///home/conda/feedstock_root/build_artifacts/astroid_1695739484762/work\r\n",
      "astropy @ file:///home/conda/feedstock_root/build_artifacts/astropy_1701289853072/work\r\n",
      "astropy-iers-data @ file:///home/conda/feedstock_root/build_artifacts/astropy-iers-data_1706498672322/work\r\n",
      "asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1698341106958/work\r\n",
      "async-lru @ file:///home/conda/feedstock_root/build_artifacts/async-lru_1690563019058/work\r\n",
      "async-timeout @ file:///home/conda/feedstock_root/build_artifacts/async-timeout_1691763562544/work\r\n",
      "atomicwrites @ file:///home/conda/feedstock_root/build_artifacts/atomicwrites_1657325823582/work\r\n",
      "attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1704011227531/work\r\n",
      "autopep8 @ file:///home/conda/feedstock_root/build_artifacts/autopep8_1693061251004/work\r\n",
      "autovizwidget @ file:///home/conda/feedstock_root/build_artifacts/autovizwidget_1694633627542/work\r\n",
      "awscli==1.32.50\r\n",
      "Babel @ file:///home/conda/feedstock_root/build_artifacts/babel_1702422572539/work\r\n",
      "beautifulsoup4 @ file:///home/conda/feedstock_root/build_artifacts/beautifulsoup4_1705564648255/work\r\n",
      "binaryornot==0.4.4\r\n",
      "bitarray @ file:///home/conda/feedstock_root/build_artifacts/bitarray_1704563349656/work\r\n",
      "black @ file:///home/conda/feedstock_root/build_artifacts/black-recipe_1706510392927/work\r\n",
      "bleach @ file:///home/conda/feedstock_root/build_artifacts/bleach_1696630167146/work\r\n",
      "blinker @ file:///home/conda/feedstock_root/build_artifacts/blinker_1698890160476/work\r\n",
      "bokeh @ file:///home/conda/feedstock_root/build_artifacts/bokeh_1706215790147/work\r\n",
      "boto3==1.34.50\r\n",
      "botocore==1.34.50\r\n",
      "Bottleneck @ file:///home/conda/feedstock_root/build_artifacts/bottleneck_1696017913939/work\r\n",
      "Brotli @ file:///home/conda/feedstock_root/build_artifacts/brotli-split_1695989787169/work\r\n",
      "brotlipy @ file:///home/conda/feedstock_root/build_artifacts/brotlipy_1695621686607/work\r\n",
      "cached-property @ file:///home/conda/feedstock_root/build_artifacts/cached_property_1615209429212/work\r\n",
      "certifi==2024.2.2\r\n",
      "cffi @ file:///home/conda/feedstock_root/build_artifacts/cffi_1696001684923/work\r\n",
      "chardet @ file:///home/conda/feedstock_root/build_artifacts/chardet_1695468598188/work\r\n",
      "charset-normalizer @ file:///home/conda/feedstock_root/build_artifacts/charset-normalizer_1698833585322/work\r\n",
      "click @ file:///home/conda/feedstock_root/build_artifacts/click_1692311806742/work\r\n",
      "cloudpickle==2.2.1\r\n",
      "colorama==0.4.4\r\n",
      "comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1704278392174/work\r\n",
      "contextlib2==21.6.0\r\n",
      "contourpy @ file:///home/conda/feedstock_root/build_artifacts/contourpy_1699041363598/work\r\n",
      "cookiecutter @ file:///home/conda/feedstock_root/build_artifacts/cookiecutter_1700591923152/work\r\n",
      "coverage @ file:///home/conda/feedstock_root/build_artifacts/coverage_1706301671255/work\r\n",
      "cryptography @ file:///home/conda/feedstock_root/build_artifacts/cryptography-split_1706658599627/work\r\n",
      "cycler @ file:///home/conda/feedstock_root/build_artifacts/cycler_1696677705766/work\r\n",
      "Cython @ file:///home/conda/feedstock_root/build_artifacts/cython_1706274771717/work\r\n",
      "cytoolz @ file:///home/conda/feedstock_root/build_artifacts/cytoolz_1695545144718/work\r\n",
      "dask @ file:///home/conda/feedstock_root/build_artifacts/dask-core_1706311375733/work\r\n",
      "debugpy @ file:///home/conda/feedstock_root/build_artifacts/debugpy_1695534290310/work\r\n",
      "decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work\r\n",
      "defusedxml @ file:///home/conda/feedstock_root/build_artifacts/defusedxml_1615232257335/work\r\n",
      "diff-match-patch @ file:///home/conda/feedstock_root/build_artifacts/diff-match-patch_1683670697993/work\r\n",
      "dill @ file:///home/conda/feedstock_root/build_artifacts/dill_1706434688412/work\r\n",
      "distributed @ file:///home/conda/feedstock_root/build_artifacts/distributed_1706315353433/work\r\n",
      "docker==6.1.3\r\n",
      "docstring-to-markdown @ file:///home/conda/feedstock_root/build_artifacts/docstring-to-markdown_1697065579403/work\r\n",
      "docutils==0.16\r\n",
      "dparse==0.6.3\r\n",
      "entrypoints @ file:///home/conda/feedstock_root/build_artifacts/entrypoints_1643888246732/work\r\n",
      "et-xmlfile @ file:///home/conda/feedstock_root/build_artifacts/et_xmlfile_1674664118162/work\r\n",
      "exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1704921103267/work\r\n",
      "executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1698579936712/work\r\n",
      "fastcache @ file:///home/conda/feedstock_root/build_artifacts/fastcache_1695986192626/work\r\n",
      "fastjsonschema @ file:///home/conda/feedstock_root/build_artifacts/python-fastjsonschema_1703780968325/work/dist\r\n",
      "filelock @ file:///home/conda/feedstock_root/build_artifacts/filelock_1698714947081/work\r\n",
      "flake8 @ file:///home/conda/feedstock_root/build_artifacts/flake8_1669396691980/work\r\n",
      "Flask @ file:///home/conda/feedstock_root/build_artifacts/flask_1705682375678/work\r\n",
      "Flask-Cors @ file:///home/conda/feedstock_root/build_artifacts/flask-cors_1687783436674/work\r\n",
      "fonttools @ file:///home/conda/feedstock_root/build_artifacts/fonttools_1704979835861/work\r\n",
      "fqdn @ file:///home/conda/feedstock_root/build_artifacts/fqdn_1638810296540/work/dist\r\n",
      "frozenlist @ file:///home/conda/feedstock_root/build_artifacts/frozenlist_1702645481127/work\r\n",
      "fsspec @ file:///home/conda/feedstock_root/build_artifacts/fsspec_1702335961905/work\r\n",
      "future @ file:///home/conda/feedstock_root/build_artifacts/future_1673596611778/work\r\n",
      "gevent @ file:///home/conda/feedstock_root/build_artifacts/gevent_1696750251337/work\r\n",
      "gmpy2 @ file:///home/conda/feedstock_root/build_artifacts/gmpy2_1666808654411/work\r\n",
      "google-pasta==0.2.0\r\n",
      "greenlet @ file:///home/conda/feedstock_root/build_artifacts/greenlet_1703201576006/work\r\n",
      "h5py @ file:///home/conda/feedstock_root/build_artifacts/h5py_1702471429234/work\r\n",
      "hdijupyterutils @ file:///home/conda/feedstock_root/build_artifacts/hdijupyterutils_1694633580855/work\r\n",
      "idna @ file:///home/conda/feedstock_root/build_artifacts/idna_1701026962277/work\r\n",
      "imagecodecs @ file:///home/conda/feedstock_root/build_artifacts/imagecodecs_1704019731389/work\r\n",
      "imageio @ file:///home/conda/feedstock_root/build_artifacts/imageio_1702571712725/work\r\n",
      "imagesize @ file:///home/conda/feedstock_root/build_artifacts/imagesize_1656939531508/work\r\n",
      "immutables @ file:///home/conda/feedstock_root/build_artifacts/immutables_1695646628643/work\r\n",
      "importlib-metadata==6.11.0\r\n",
      "importlib-resources @ file:///home/conda/feedstock_root/build_artifacts/importlib_resources_1699364556997/work\r\n",
      "inflection @ file:///home/conda/feedstock_root/build_artifacts/inflection_1598089801258/work\r\n",
      "iniconfig @ file:///home/conda/feedstock_root/build_artifacts/iniconfig_1673103042956/work\r\n",
      "intervaltree @ file:///home/conda/feedstock_root/build_artifacts/intervaltree_1683532206518/work\r\n",
      "ipykernel @ file:///home/conda/feedstock_root/build_artifacts/ipykernel_1705417941265/work\r\n",
      "ipython @ file:///home/conda/feedstock_root/build_artifacts/ipython_1706795662110/work\r\n",
      "ipython-genutils==0.2.0\r\n",
      "ipywidgets @ file:///home/conda/feedstock_root/build_artifacts/ipywidgets_1694607144474/work\r\n",
      "isoduration @ file:///home/conda/feedstock_root/build_artifacts/isoduration_1638811571363/work/dist\r\n",
      "isort @ file:///home/conda/feedstock_root/build_artifacts/isort_1702518492027/work\r\n",
      "itsdangerous @ file:///home/conda/feedstock_root/build_artifacts/itsdangerous_1648147185463/work\r\n",
      "jaraco.classes @ file:///home/conda/feedstock_root/build_artifacts/jaraco.classes_1689112411129/work\r\n",
      "jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1669134318875/work\r\n",
      "jeepney @ file:///home/conda/feedstock_root/build_artifacts/jeepney_1649085214306/work\r\n",
      "jellyfish @ file:///home/conda/feedstock_root/build_artifacts/jellyfish_1700261197714/work\r\n",
      "Jinja2 @ file:///home/conda/feedstock_root/build_artifacts/jinja2_1704966972576/work\r\n",
      "jmespath @ file:///home/conda/feedstock_root/build_artifacts/jmespath_1655568249366/work\r\n",
      "joblib @ file:///home/conda/feedstock_root/build_artifacts/joblib_1691577114857/work\r\n",
      "json5 @ file:///home/conda/feedstock_root/build_artifacts/json5_1688248289187/work\r\n",
      "jsonpointer @ file:///home/conda/feedstock_root/build_artifacts/jsonpointer_1695397238043/work\r\n",
      "jsonschema @ file:///home/conda/feedstock_root/build_artifacts/jsonschema-meta_1705707496704/work\r\n",
      "jsonschema-specifications @ file:///tmp/tmpkv1z7p57/src\r\n",
      "jupyter @ file:///home/conda/feedstock_root/build_artifacts/jupyter_1696255489086/work\r\n",
      "jupyter-console @ file:///home/conda/feedstock_root/build_artifacts/jupyter_console_1678118109161/work\r\n",
      "jupyter-events @ file:///home/conda/feedstock_root/build_artifacts/jupyter_events_1699285872613/work\r\n",
      "jupyter-lsp @ file:///home/conda/feedstock_root/build_artifacts/jupyter-lsp-meta_1705532074645/work/jupyter-lsp\r\n",
      "jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1699283905679/work\r\n",
      "jupyter_core @ file:///home/conda/feedstock_root/build_artifacts/jupyter_core_1704727030956/work\r\n",
      "jupyter_server @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_1705418153950/work\r\n",
      "jupyter_server_terminals @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_terminals_1706006699561/work\r\n",
      "jupyterlab @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_1706634375218/work\r\n",
      "jupyterlab-widgets @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_widgets_1694598704522/work\r\n",
      "jupyterlab_pygments @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_pygments_1700744013163/work\r\n",
      "jupyterlab_server @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_server-split_1700310846957/work\r\n",
      "keyring @ file:///home/conda/feedstock_root/build_artifacts/keyring_1699923928748/work\r\n",
      "kiwisolver @ file:///home/conda/feedstock_root/build_artifacts/kiwisolver_1695379902431/work\r\n",
      "lazy-object-proxy @ file:///home/conda/feedstock_root/build_artifacts/lazy-object-proxy_1702663550721/work\r\n",
      "lazy_loader @ file:///home/conda/feedstock_root/build_artifacts/lazy_loader_1692295373316/work\r\n",
      "llvmlite==0.41.1\r\n",
      "locket @ file:///home/conda/feedstock_root/build_artifacts/locket_1650660393415/work\r\n",
      "lz4 @ file:///home/conda/feedstock_root/build_artifacts/lz4_1704831084136/work\r\n",
      "markdown-it-py @ file:///home/conda/feedstock_root/build_artifacts/markdown-it-py_1686175045316/work\r\n",
      "MarkupSafe @ file:///home/conda/feedstock_root/build_artifacts/markupsafe_1705778934742/work\r\n",
      "matplotlib @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-suite_1700509477637/work\r\n",
      "matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1660814786464/work\r\n",
      "mccabe @ file:///home/conda/feedstock_root/build_artifacts/mccabe_1643049622439/work\r\n",
      "mdurl @ file:///home/conda/feedstock_root/build_artifacts/mdurl_1704317613764/work\r\n",
      "mistune @ file:///home/conda/feedstock_root/build_artifacts/mistune_1698947099619/work\r\n",
      "mkl-service==2.4.0\r\n",
      "mkl_fft==1.3.8\r\n",
      "mock @ file:///home/conda/feedstock_root/build_artifacts/mock_1689092066756/work\r\n",
      "more-itertools @ file:///home/conda/feedstock_root/build_artifacts/more-itertools_1704738417589/work\r\n",
      "mpmath @ file:///home/conda/feedstock_root/build_artifacts/mpmath_1678228039184/work\r\n",
      "msgpack @ file:///home/conda/feedstock_root/build_artifacts/msgpack-python_1700926504817/work\r\n",
      "multidict @ file:///home/conda/feedstock_root/build_artifacts/multidict_1696716075096/work\r\n",
      "multiprocess==0.70.16\r\n",
      "munkres==1.1.4\r\n",
      "mypy-extensions @ file:///home/conda/feedstock_root/build_artifacts/mypy_extensions_1675543315189/work\r\n",
      "nbclient @ file:///home/conda/feedstock_root/build_artifacts/nbclient_1684790896106/work\r\n",
      "nbconvert @ file:///home/conda/feedstock_root/build_artifacts/nbconvert-meta_1705418470095/work\r\n",
      "nbformat @ file:///home/conda/feedstock_root/build_artifacts/nbformat_1690814868471/work\r\n",
      "nest_asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1705850609492/work\r\n",
      "networkx @ file:///home/conda/feedstock_root/build_artifacts/networkx_1698504735452/work\r\n",
      "nltk @ file:///home/conda/feedstock_root/build_artifacts/nltk_1672696305909/work\r\n",
      "nose @ file:///home/conda/feedstock_root/build_artifacts/nose_1602434998960/work\r\n",
      "notebook @ file:///home/conda/feedstock_root/build_artifacts/notebook_1705686947570/work\r\n",
      "notebook_shim @ file:///home/conda/feedstock_root/build_artifacts/notebook-shim_1682360583588/work\r\n",
      "numba @ file:///home/conda/feedstock_root/build_artifacts/numba_1699456635499/work\r\n",
      "numexpr @ file:///home/conda/feedstock_root/build_artifacts/numexpr_1703356700084/work\r\n",
      "numpy @ file:///home/conda/feedstock_root/build_artifacts/numpy_1653325310407/work\r\n",
      "numpydoc @ file:///home/conda/feedstock_root/build_artifacts/numpydoc_1702057563287/work\r\n",
      "openpyxl @ file:///home/conda/feedstock_root/build_artifacts/openpyxl_1695464693876/work\r\n",
      "overrides @ file:///home/conda/feedstock_root/build_artifacts/overrides_1706394519472/work\r\n",
      "packaging==21.3\r\n",
      "pandas @ file:///home/conda/feedstock_root/build_artifacts/pandas_1705728427355/work\r\n",
      "pandocfilters @ file:///home/conda/feedstock_root/build_artifacts/pandocfilters_1631603243851/work\r\n",
      "parso @ file:///home/conda/feedstock_root/build_artifacts/parso_1638334955874/work\r\n",
      "partd @ file:///home/conda/feedstock_root/build_artifacts/partd_1695667515973/work\r\n",
      "path @ file:///home/conda/feedstock_root/build_artifacts/path_1703546324061/work\r\n",
      "pathlib2 @ file:///home/conda/feedstock_root/build_artifacts/pathlib2_1695464058821/work\r\n",
      "pathos==0.3.2\r\n",
      "pathspec @ file:///home/conda/feedstock_root/build_artifacts/pathspec_1702249949303/work\r\n",
      "patsy @ file:///home/conda/feedstock_root/build_artifacts/patsy_1704469236901/work\r\n",
      "pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1706113125309/work\r\n",
      "pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1602536217715/work\r\n",
      "pillow @ file:///home/conda/feedstock_root/build_artifacts/pillow_1704252020178/work\r\n",
      "pkginfo @ file:///home/conda/feedstock_root/build_artifacts/pkginfo_1673281726124/work\r\n",
      "pkgutil_resolve_name @ file:///home/conda/feedstock_root/build_artifacts/pkgutil-resolve-name_1694617248815/work\r\n",
      "platformdirs @ file:///home/conda/feedstock_root/build_artifacts/platformdirs_1706713388748/work\r\n",
      "plotly @ file:///home/conda/feedstock_root/build_artifacts/plotly_1698272730927/work\r\n",
      "pluggy @ file:///home/conda/feedstock_root/build_artifacts/pluggy_1706116770704/work\r\n",
      "ply==3.11\r\n",
      "pox==0.3.4\r\n",
      "ppft==1.7.6.8\r\n",
      "prometheus-client @ file:///home/conda/feedstock_root/build_artifacts/prometheus_client_1700579315247/work\r\n",
      "prompt-toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1702399386289/work\r\n",
      "protobuf==4.25.3\r\n",
      "psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1705722392846/work\r\n",
      "psycopg2 @ file:///home/conda/feedstock_root/build_artifacts/psycopg2-split_1701737488821/work\r\n",
      "psycopg2-binary @ file:///home/conda/feedstock_root/build_artifacts/psycopg2-split_1701737547976/work/psycopg2-binary\r\n",
      "ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1609419310487/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl\r\n",
      "pure-eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1642875951954/work\r\n",
      "py-cpuinfo @ file:///home/conda/feedstock_root/build_artifacts/py-cpuinfo_1666774466606/work\r\n",
      "py4j==0.10.9.5\r\n",
      "pyarrow==15.0.0\r\n",
      "pyarrow-hotfix @ file:///home/conda/feedstock_root/build_artifacts/pyarrow-hotfix_1700596371886/work\r\n",
      "pyasn1==0.5.1\r\n",
      "pycodestyle @ file:///home/conda/feedstock_root/build_artifacts/pycodestyle_1669306857274/work\r\n",
      "pycosat @ file:///home/conda/feedstock_root/build_artifacts/pycosat_1696355758174/work\r\n",
      "pycparser @ file:///home/conda/feedstock_root/build_artifacts/pycparser_1636257122734/work\r\n",
      "pycryptodome @ file:///home/conda/feedstock_root/build_artifacts/pycryptodome_1704930718567/work\r\n",
      "pycurl==7.45.1\r\n",
      "pydocstyle @ file:///home/conda/feedstock_root/build_artifacts/pydocstyle_1673997487070/work\r\n",
      "pyerfa @ file:///home/conda/feedstock_root/build_artifacts/pyerfa_1697801090826/work\r\n",
      "pyflakes @ file:///home/conda/feedstock_root/build_artifacts/pyflakes_1669319921641/work\r\n",
      "Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1700607939962/work\r\n",
      "pykerberos @ file:///home/conda/feedstock_root/build_artifacts/pykerberos_1695558046383/work\r\n",
      "pylint @ file:///home/conda/feedstock_root/build_artifacts/pylint_1696171682664/work\r\n",
      "pylint-venv @ file:///home/conda/feedstock_root/build_artifacts/pylint-venv_1698219336631/work\r\n",
      "pyls-spyder @ file:///home/conda/feedstock_root/build_artifacts/pyls-spyder_1619747398504/work\r\n",
      "pynvml @ file:///home/conda/feedstock_root/build_artifacts/pynvml_1676513396402/work\r\n",
      "pyodbc @ file:///home/conda/feedstock_root/build_artifacts/pyodbc_1697223139819/work\r\n",
      "pyOpenSSL @ file:///home/conda/feedstock_root/build_artifacts/pyopenssl_1706660063483/work\r\n",
      "pyparsing @ file:///home/conda/feedstock_root/build_artifacts/pyparsing_1690737849915/work\r\n",
      "PyQt5==5.15.9\r\n",
      "PyQt5-sip==12.12.2\r\n",
      "PyQtWebEngine==5.15.4\r\n",
      "pyrsistent @ file:///home/conda/feedstock_root/build_artifacts/pyrsistent_1698753827123/work\r\n",
      "PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1661604839144/work\r\n",
      "pyspark==3.3.0\r\n",
      "pyspnego @ file:///home/conda/feedstock_root/build_artifacts/pyspnego_1696277744607/work\r\n",
      "pytest @ file:///home/conda/feedstock_root/build_artifacts/pytest_1706447941722/work\r\n",
      "python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/python-dateutil_1626286286081/work\r\n",
      "python-json-logger @ file:///home/conda/feedstock_root/build_artifacts/python-json-logger_1677079630776/work\r\n",
      "python-lsp-black @ file:///home/conda/feedstock_root/build_artifacts/python-lsp-black_1702956932456/work\r\n",
      "python-lsp-jsonrpc @ file:///home/conda/feedstock_root/build_artifacts/python-lsp-jsonrpc_1695528365348/work\r\n",
      "python-lsp-server @ file:///home/conda/feedstock_root/build_artifacts/python-lsp-server-meta_1688065622325/work\r\n",
      "python-slugify @ file:///home/conda/feedstock_root/build_artifacts/python-slugify-split_1706737809183/work\r\n",
      "pytoolconfig @ file:///home/conda/feedstock_root/build_artifacts/pytoolconfig_1675124745143/work\r\n",
      "pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1706549378554/work\r\n",
      "PyWavelets @ file:///home/conda/feedstock_root/build_artifacts/pywavelets_1695567558330/work\r\n",
      "pyxdg @ file:///home/conda/feedstock_root/build_artifacts/pyxdg_1654536799286/work\r\n",
      "PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1695373428874/work\r\n",
      "pyzmq @ file:///home/conda/feedstock_root/build_artifacts/pyzmq_1701783169502/work\r\n",
      "QDarkStyle @ file:///home/conda/feedstock_root/build_artifacts/qdarkstyle_1654095878638/work\r\n",
      "qstylizer @ file:///home/conda/feedstock_root/build_artifacts/qstylizer_1662244505808/work/dist/qstylizer-0.2.2-py2.py3-none-any.whl\r\n",
      "QtAwesome @ file:///home/conda/feedstock_root/build_artifacts/qtawesome_1702340850976/work\r\n",
      "qtconsole @ file:///home/conda/feedstock_root/build_artifacts/qtconsole-base_1693604303222/work\r\n",
      "QtPy @ file:///home/conda/feedstock_root/build_artifacts/qtpy_1698112029416/work\r\n",
      "referencing @ file:///home/conda/feedstock_root/build_artifacts/referencing_1706711412823/work\r\n",
      "regex @ file:///home/conda/feedstock_root/build_artifacts/regex_1703393490683/work\r\n",
      "requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1684774241324/work\r\n",
      "requests-kerberos @ file:///home/conda/feedstock_root/build_artifacts/requests-kerberos_1697118166865/work\r\n",
      "rfc3339-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3339-validator_1638811747357/work\r\n",
      "rfc3986-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3986-validator_1598024191506/work\r\n",
      "rich @ file:///home/conda/feedstock_root/build_artifacts/rich-split_1700160075651/work/dist\r\n",
      "rope @ file:///home/conda/feedstock_root/build_artifacts/rope_1705558091413/work\r\n",
      "rpds-py @ file:///home/conda/feedstock_root/build_artifacts/rpds-py_1705159800683/work\r\n",
      "rsa==4.7.2\r\n",
      "Rtree @ file:///home/conda/feedstock_root/build_artifacts/rtree_1705697867335/work\r\n",
      "ruamel-yaml-conda @ file:///home/conda/feedstock_root/build_artifacts/ruamel_yaml_1695546328261/work\r\n",
      "ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1699007337104/work\r\n",
      "ruamel.yaml.clib @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml.clib_1695996839082/work\r\n",
      "s3fs==0.4.2\r\n",
      "s3transfer @ file:///home/conda/feedstock_root/build_artifacts/s3transfer_1703197439685/work\r\n",
      "sagemaker==2.209.0\r\n",
      "sagemaker_pyspark==1.4.5\r\n",
      "schema==0.7.5\r\n",
      "scikit-image @ file:///home/conda/feedstock_root/build_artifacts/scikit-image_1697028626781/work/dist/scikit_image-0.22.0-cp310-cp310-linux_x86_64.whl#sha256=6c8036af11a5f2bf6fdd54b2feaf836dde1a27bdf20d80edad07f7770e4a9fd0\r\n",
      "scikit-learn @ file:///home/conda/feedstock_root/build_artifacts/scikit-learn_1705657307465/work\r\n",
      "scipy @ file:///home/conda/feedstock_root/build_artifacts/scipy-split_1706041478791/work/dist/scipy-1.12.0-cp310-cp310-linux_x86_64.whl#sha256=2f498a77f15907a2dfec49ca55a78d94633a6c8be7ef034a116c41cbea5c4241\r\n",
      "seaborn @ file:///home/conda/feedstock_root/build_artifacts/seaborn-split_1706340836595/work\r\n",
      "SecretStorage @ file:///home/conda/feedstock_root/build_artifacts/secretstorage_1695551734488/work\r\n",
      "Send2Trash @ file:///home/conda/feedstock_root/build_artifacts/send2trash_1682601222253/work\r\n",
      "shap @ file:///home/conda/feedstock_root/build_artifacts/shap_1704716387123/work\r\n",
      "sip @ file:///home/conda/feedstock_root/build_artifacts/sip_1697300428978/work\r\n",
      "six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work\r\n",
      "slicer @ file:///home/conda/feedstock_root/build_artifacts/slicer_1608146800664/work\r\n",
      "smdebug-rulesconfig==1.0.1\r\n",
      "sniffio @ file:///home/conda/feedstock_root/build_artifacts/sniffio_1662051266223/work\r\n",
      "snowballstemmer @ file:///home/conda/feedstock_root/build_artifacts/snowballstemmer_1637143057757/work\r\n",
      "sortedcontainers @ file:///home/conda/feedstock_root/build_artifacts/sortedcontainers_1621217038088/work\r\n",
      "soupsieve @ file:///home/conda/feedstock_root/build_artifacts/soupsieve_1693929250441/work\r\n",
      "sparkmagic @ file:///home/conda/feedstock_root/build_artifacts/sparkmagic_1694633601704/work/sparkmagic\r\n",
      "Sphinx @ file:///home/conda/feedstock_root/build_artifacts/sphinx_1694647393084/work\r\n",
      "sphinxcontrib-applehelp @ file:///home/conda/feedstock_root/build_artifacts/sphinxcontrib-applehelp_1705126298355/work\r\n",
      "sphinxcontrib-devhelp @ file:///home/conda/feedstock_root/build_artifacts/sphinxcontrib-devhelp_1705126010477/work\r\n",
      "sphinxcontrib-htmlhelp @ file:///home/conda/feedstock_root/build_artifacts/sphinxcontrib-htmlhelp_1705118152391/work\r\n",
      "sphinxcontrib-jsmath @ file:///home/conda/feedstock_root/build_artifacts/sphinxcontrib-jsmath_1691604704163/work\r\n",
      "sphinxcontrib-qthelp @ file:///home/conda/feedstock_root/build_artifacts/sphinxcontrib-qthelp_1705126152907/work\r\n",
      "sphinxcontrib-serializinghtml @ file:///home/conda/feedstock_root/build_artifacts/sphinxcontrib-serializinghtml_1705118225549/work\r\n",
      "sphinxcontrib-websupport @ file:///home/conda/feedstock_root/build_artifacts/sphinxcontrib-websupport_1705125899521/work\r\n",
      "spyder @ file:///home/conda/feedstock_root/build_artifacts/spyder_1693435026846/work\r\n",
      "spyder-kernels @ file:///home/conda/feedstock_root/build_artifacts/spyder-kernels_1688063243300/work\r\n",
      "SQLAlchemy @ file:///home/conda/feedstock_root/build_artifacts/sqlalchemy_1704267473995/work\r\n",
      "stack-data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1669632077133/work\r\n",
      "statsmodels @ file:///home/conda/feedstock_root/build_artifacts/statsmodels_1702575356319/work\r\n",
      "sympy @ file:///home/conda/feedstock_root/build_artifacts/sympy_1684180540116/work\r\n",
      "tables @ file:///home/conda/feedstock_root/build_artifacts/pytables_1701702211874/work\r\n",
      "tabulate @ file:///home/conda/feedstock_root/build_artifacts/tabulate_1665138452165/work\r\n",
      "tblib==2.0.0\r\n",
      "tenacity @ file:///home/conda/feedstock_root/build_artifacts/tenacity_1692026804430/work\r\n",
      "terminado @ file:///home/conda/feedstock_root/build_artifacts/terminado_1699810101464/work\r\n",
      "testpath @ file:///home/conda/feedstock_root/build_artifacts/testpath_1645693042223/work\r\n",
      "text-unidecode @ file:///home/conda/feedstock_root/build_artifacts/text-unidecode_1694707102786/work\r\n",
      "textdistance @ file:///home/conda/feedstock_root/build_artifacts/textdistance_1663527496115/work\r\n",
      "threadpoolctl @ file:///home/conda/feedstock_root/build_artifacts/threadpoolctl_1689261241048/work\r\n",
      "three-merge @ file:///home/conda/feedstock_root/build_artifacts/three-merge_1595515817927/work\r\n",
      "tifffile @ file:///home/conda/feedstock_root/build_artifacts/tifffile_1706606216450/work\r\n",
      "tinycss2 @ file:///home/conda/feedstock_root/build_artifacts/tinycss2_1666100256010/work\r\n",
      "toml @ file:///home/conda/feedstock_root/build_artifacts/toml_1604308577558/work\r\n",
      "tomli @ file:///home/conda/feedstock_root/build_artifacts/tomli_1644342247877/work\r\n",
      "tomlkit @ file:///home/conda/feedstock_root/build_artifacts/tomlkit_1700046708542/work\r\n",
      "toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1706112571092/work\r\n",
      "tornado @ file:///home/conda/feedstock_root/build_artifacts/tornado_1695373560918/work\r\n",
      "tqdm @ file:///home/conda/feedstock_root/build_artifacts/tqdm_1691671248568/work\r\n",
      "traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1704212992681/work\r\n",
      "typed-ast @ file:///home/conda/feedstock_root/build_artifacts/typed-ast_1695409894288/work\r\n",
      "types-python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/types-python-dateutil_1704512562698/work\r\n",
      "typing-utils @ file:///home/conda/feedstock_root/build_artifacts/typing_utils_1622899189314/work\r\n",
      "typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/typing_extensions_1702176139754/work\r\n",
      "tzdata @ file:///home/conda/feedstock_root/build_artifacts/python-tzdata_1703878702368/work\r\n",
      "ujson @ file:///home/conda/feedstock_root/build_artifacts/ujson_1702256697606/work\r\n",
      "unicodedata2 @ file:///home/conda/feedstock_root/build_artifacts/unicodedata2_1695847980273/work\r\n",
      "uri-template @ file:///home/conda/feedstock_root/build_artifacts/uri-template_1688655812972/work/dist\r\n",
      "urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1697813446430/work\r\n",
      "watchdog @ file:///home/conda/feedstock_root/build_artifacts/watchdog_1695395236885/work\r\n",
      "wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1704731205417/work\r\n",
      "webcolors @ file:///home/conda/feedstock_root/build_artifacts/webcolors_1679900785843/work\r\n",
      "webencodings @ file:///home/conda/feedstock_root/build_artifacts/webencodings_1694681268211/work\r\n",
      "websocket-client @ file:///home/conda/feedstock_root/build_artifacts/websocket-client_1701630677416/work\r\n",
      "Werkzeug @ file:///home/conda/feedstock_root/build_artifacts/werkzeug_1698235201373/work\r\n",
      "whatthepatch @ file:///home/conda/feedstock_root/build_artifacts/whatthepatch_1683396758362/work\r\n",
      "widgetsnbextension @ file:///home/conda/feedstock_root/build_artifacts/widgetsnbextension_1694598693908/work\r\n",
      "wrapt @ file:///home/conda/feedstock_root/build_artifacts/wrapt_1699532811524/work\r\n",
      "wurlitzer @ file:///home/conda/feedstock_root/build_artifacts/wurlitzer_1669944596833/work\r\n",
      "XlsxWriter @ file:///home/conda/feedstock_root/build_artifacts/xlsxwriter_1699211250539/work\r\n",
      "xyzservices @ file:///home/conda/feedstock_root/build_artifacts/xyzservices_1698325309404/work\r\n",
      "yapf @ file:///home/conda/feedstock_root/build_artifacts/yapf_1690387939953/work\r\n",
      "yarl @ file:///home/conda/feedstock_root/build_artifacts/yarl_1705508292061/work\r\n",
      "zict @ file:///home/conda/feedstock_root/build_artifacts/zict_1681770155528/work\r\n",
      "zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1695255097490/work\r\n",
      "zope.event @ file:///home/conda/feedstock_root/build_artifacts/zope.event_1687705558811/work\r\n",
      "zope.interface @ file:///home/conda/feedstock_root/build_artifacts/zope.interface_1696523477995/work\r\n"
     ]
    }
   ],
   "source": [
    "!pip freeze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4cb8d09d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import boto3\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "a259d987",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(140, 5)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rain = pd.read_csv('s3://dev-cucumbers/path/to/folder3/arima_timeseries_run_123456789012_chunk_0.csv')\n",
    "rain.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "21a4faa9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rain['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "ba00d8ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(213, 7)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "elisa = pd.read_csv('s3://dev-cucumbers/eimpara/pipelines/moments/CLUSTERING_arima_cross_test_run_for_Rain.csv')\n",
    "elisa.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "4dee0f70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>TRANSACTION_COUNTRY_CODE</th>\n",
       "      <th>avg_streams_train</th>\n",
       "      <th>avg_streams_test</th>\n",
       "      <th>sum_forecast_errors</th>\n",
       "      <th>performance_test_period</th>\n",
       "      <th>slope_test_period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>GB</td>\n",
       "      <td>3595.007519</td>\n",
       "      <td>3426.000000</td>\n",
       "      <td>2151.331</td>\n",
       "      <td>significantly_better_than_predicted</td>\n",
       "      <td>down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AEA2D2300306</td>\n",
       "      <td>GB</td>\n",
       "      <td>109.977444</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>27.629</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AEA2W2369250</td>\n",
       "      <td>GB</td>\n",
       "      <td>150.736842</td>\n",
       "      <td>119.142857</td>\n",
       "      <td>-79.177</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ARA340900073</td>\n",
       "      <td>GB</td>\n",
       "      <td>740.278195</td>\n",
       "      <td>609.571429</td>\n",
       "      <td>997.238</td>\n",
       "      <td>marginally_better_than_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ARBAA1600002</td>\n",
       "      <td>GB</td>\n",
       "      <td>2360.887218</td>\n",
       "      <td>2113.000000</td>\n",
       "      <td>291.402</td>\n",
       "      <td>marginally_better_than_predicted</td>\n",
       "      <td>up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>ATN262206005</td>\n",
       "      <td>GB</td>\n",
       "      <td>393.443609</td>\n",
       "      <td>227.714286</td>\n",
       "      <td>-180.212</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>ATN262206006</td>\n",
       "      <td>GB</td>\n",
       "      <td>321.706767</td>\n",
       "      <td>195.285714</td>\n",
       "      <td>-326.185</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>ATN262210902</td>\n",
       "      <td>GB</td>\n",
       "      <td>208.353383</td>\n",
       "      <td>192.714286</td>\n",
       "      <td>-221.059</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>ATN262210904</td>\n",
       "      <td>GB</td>\n",
       "      <td>760.676692</td>\n",
       "      <td>678.571429</td>\n",
       "      <td>-695.172</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>GB</td>\n",
       "      <td>218.639098</td>\n",
       "      <td>200.714286</td>\n",
       "      <td>44.450</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_down_trend</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>213 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             ISRC TRANSACTION_COUNTRY_CODE  avg_streams_train  \\\n",
       "0    AEA0Q2149133                       GB        3595.007519   \n",
       "1    AEA2D2300306                       GB         109.977444   \n",
       "2    AEA2W2369250                       GB         150.736842   \n",
       "3    ARA340900073                       GB         740.278195   \n",
       "4    ARBAA1600002                       GB        2360.887218   \n",
       "..            ...                      ...                ...   \n",
       "208  ATN262206005                       GB         393.443609   \n",
       "209  ATN262206006                       GB         321.706767   \n",
       "210  ATN262210902                       GB         208.353383   \n",
       "211  ATN262210904                       GB         760.676692   \n",
       "212  ATN262210905                       GB         218.639098   \n",
       "\n",
       "     avg_streams_test  sum_forecast_errors  \\\n",
       "0         3426.000000             2151.331   \n",
       "1           61.000000               27.629   \n",
       "2          119.142857              -79.177   \n",
       "3          609.571429              997.238   \n",
       "4         2113.000000              291.402   \n",
       "..                ...                  ...   \n",
       "208        227.714286             -180.212   \n",
       "209        195.285714             -326.185   \n",
       "210        192.714286             -221.059   \n",
       "211        678.571429             -695.172   \n",
       "212        200.714286               44.450   \n",
       "\n",
       "                 performance_test_period  slope_test_period  \n",
       "0    significantly_better_than_predicted         down_trend  \n",
       "1                           as_predicted  slight_down_trend  \n",
       "2                           as_predicted    slight_up_trend  \n",
       "3       marginally_better_than_predicted    slight_up_trend  \n",
       "4       marginally_better_than_predicted           up_trend  \n",
       "..                                   ...                ...  \n",
       "208                         as_predicted           up_trend  \n",
       "209                         as_predicted         down_trend  \n",
       "210                         as_predicted         down_trend  \n",
       "211                         as_predicted    slight_up_trend  \n",
       "212                         as_predicted  slight_down_trend  \n",
       "\n",
       "[213 rows x 7 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "elisa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "61b9d629",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>TRANSACTION_COUNTRY_CODE</th>\n",
       "      <th>avg_streams_train</th>\n",
       "      <th>avg_streams_test</th>\n",
       "      <th>sum_forecast_errors</th>\n",
       "      <th>performance_test_period</th>\n",
       "      <th>slope_test_period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>GB</td>\n",
       "      <td>3595.007519</td>\n",
       "      <td>3426.000000</td>\n",
       "      <td>2151.247</td>\n",
       "      <td>significantly_better_than_predicted</td>\n",
       "      <td>down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AEA2D2300306</td>\n",
       "      <td>GB</td>\n",
       "      <td>109.977444</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>27.629</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AEA2W2369250</td>\n",
       "      <td>GB</td>\n",
       "      <td>150.736842</td>\n",
       "      <td>119.142857</td>\n",
       "      <td>-79.177</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ARA340900073</td>\n",
       "      <td>GB</td>\n",
       "      <td>740.278195</td>\n",
       "      <td>609.571429</td>\n",
       "      <td>997.239</td>\n",
       "      <td>significantly_better_than_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ARBAA1600002</td>\n",
       "      <td>GB</td>\n",
       "      <td>2360.887218</td>\n",
       "      <td>2113.000000</td>\n",
       "      <td>291.402</td>\n",
       "      <td>marginally_better_than_predicted</td>\n",
       "      <td>up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>ATN262206005</td>\n",
       "      <td>GB</td>\n",
       "      <td>393.443609</td>\n",
       "      <td>227.714286</td>\n",
       "      <td>-180.212</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>ATN262206006</td>\n",
       "      <td>GB</td>\n",
       "      <td>321.706767</td>\n",
       "      <td>195.285714</td>\n",
       "      <td>-326.185</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>ATN262210902</td>\n",
       "      <td>GB</td>\n",
       "      <td>208.353383</td>\n",
       "      <td>192.714286</td>\n",
       "      <td>-221.059</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>ATN262210904</td>\n",
       "      <td>GB</td>\n",
       "      <td>760.676692</td>\n",
       "      <td>678.571429</td>\n",
       "      <td>-695.172</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>GB</td>\n",
       "      <td>218.639098</td>\n",
       "      <td>200.714286</td>\n",
       "      <td>44.450</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_down_trend</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            ISRC TRANSACTION_COUNTRY_CODE  avg_streams_train  \\\n",
       "0   AEA0Q2149133                       GB        3595.007519   \n",
       "1   AEA2D2300306                       GB         109.977444   \n",
       "2   AEA2W2369250                       GB         150.736842   \n",
       "3   ARA340900073                       GB         740.278195   \n",
       "4   ARBAA1600002                       GB        2360.887218   \n",
       "35  ATN262206005                       GB         393.443609   \n",
       "36  ATN262206006                       GB         321.706767   \n",
       "37  ATN262210902                       GB         208.353383   \n",
       "38  ATN262210904                       GB         760.676692   \n",
       "39  ATN262210905                       GB         218.639098   \n",
       "\n",
       "    avg_streams_test  sum_forecast_errors  \\\n",
       "0        3426.000000             2151.247   \n",
       "1          61.000000               27.629   \n",
       "2         119.142857              -79.177   \n",
       "3         609.571429              997.239   \n",
       "4        2113.000000              291.402   \n",
       "35        227.714286             -180.212   \n",
       "36        195.285714             -326.185   \n",
       "37        192.714286             -221.059   \n",
       "38        678.571429             -695.172   \n",
       "39        200.714286               44.450   \n",
       "\n",
       "                performance_test_period  slope_test_period  \n",
       "0   significantly_better_than_predicted         down_trend  \n",
       "1                          as_predicted  slight_down_trend  \n",
       "2                          as_predicted    slight_up_trend  \n",
       "3   significantly_better_than_predicted    slight_up_trend  \n",
       "4      marginally_better_than_predicted           up_trend  \n",
       "35                         as_predicted           up_trend  \n",
       "36                         as_predicted         down_trend  \n",
       "37                         as_predicted         down_trend  \n",
       "38                         as_predicted    slight_up_trend  \n",
       "39                         as_predicted  slight_down_trend  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rain[rain['ISRC'].isin(['AEA0Q2149133', 'AEA2D2300306', 'AEA2W2369250', 'ARA340900073', \n",
    "                        'ARBAA1600002', 'ATN262206005', 'ATN262206006', 'ATN262210902', 'ATN262210904', 'ATN262210905'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7175db05",
   "metadata": {},
   "outputs": [],
   "source": [
    "arima_cross_run_123456789012_chunk_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d74bd7b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2181 entries, 0 to 2180\n",
      "Data columns (total 7 columns):\n",
      " #   Column                    Non-Null Count  Dtype  \n",
      "---  ------                    --------------  -----  \n",
      " 0   ISRC                      2181 non-null   object \n",
      " 1   TRANSACTION_COUNTRY_CODE  2181 non-null   object \n",
      " 2   avg_streams_train         2181 non-null   float64\n",
      " 3   avg_streams_test          2181 non-null   float64\n",
      " 4   sum_forecast_errors       2181 non-null   float64\n",
      " 5   performance_test_period   2181 non-null   object \n",
      " 6   slope_test_period         2181 non-null   object \n",
      "dtypes: float64(3), object(4)\n",
      "memory usage: 119.4+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "93d44e20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>TRANSACTION_COUNTRY_CODE</th>\n",
       "      <th>avg_streams_train</th>\n",
       "      <th>avg_streams_test</th>\n",
       "      <th>sum_forecast_errors</th>\n",
       "      <th>performance_test_period</th>\n",
       "      <th>slope_test_period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>GB</td>\n",
       "      <td>3595.007519</td>\n",
       "      <td>3426.000000</td>\n",
       "      <td>2151.331</td>\n",
       "      <td>marginally_better_than_predicted</td>\n",
       "      <td>down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AEA2D2300306</td>\n",
       "      <td>GB</td>\n",
       "      <td>109.977444</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>-85.725</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_down_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AEA2W2369250</td>\n",
       "      <td>GB</td>\n",
       "      <td>150.736842</td>\n",
       "      <td>119.142857</td>\n",
       "      <td>-26.145</td>\n",
       "      <td>as_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ARA340900073</td>\n",
       "      <td>GB</td>\n",
       "      <td>740.278195</td>\n",
       "      <td>609.571429</td>\n",
       "      <td>789.970</td>\n",
       "      <td>marginally_better_than_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ARBAA1600002</td>\n",
       "      <td>GB</td>\n",
       "      <td>2360.887218</td>\n",
       "      <td>2113.000000</td>\n",
       "      <td>895.681</td>\n",
       "      <td>marginally_better_than_predicted</td>\n",
       "      <td>up_trend</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ISRC TRANSACTION_COUNTRY_CODE  avg_streams_train  avg_streams_test  \\\n",
       "0  AEA0Q2149133                       GB        3595.007519       3426.000000   \n",
       "1  AEA2D2300306                       GB         109.977444         61.000000   \n",
       "2  AEA2W2369250                       GB         150.736842        119.142857   \n",
       "3  ARA340900073                       GB         740.278195        609.571429   \n",
       "4  ARBAA1600002                       GB        2360.887218       2113.000000   \n",
       "\n",
       "   sum_forecast_errors           performance_test_period  slope_test_period  \n",
       "0             2151.331  marginally_better_than_predicted         down_trend  \n",
       "1              -85.725                      as_predicted  slight_down_trend  \n",
       "2              -26.145                      as_predicted    slight_up_trend  \n",
       "3              789.970  marginally_better_than_predicted    slight_up_trend  \n",
       "4              895.681  marginally_better_than_predicted           up_trend  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fb127b37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "performance_test_period\n",
       "as_predicted                           1636\n",
       "marginally_better_than_predicted        544\n",
       "significantly_better_than_predicted       1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['performance_test_period'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1cf3ac63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>TRANSACTION_COUNTRY_CODE</th>\n",
       "      <th>avg_streams_train</th>\n",
       "      <th>avg_streams_test</th>\n",
       "      <th>sum_forecast_errors</th>\n",
       "      <th>performance_test_period</th>\n",
       "      <th>slope_test_period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>609</th>\n",
       "      <td>INH102308037</td>\n",
       "      <td>GB</td>\n",
       "      <td>108.165414</td>\n",
       "      <td>46.285714</td>\n",
       "      <td>1.965631e+18</td>\n",
       "      <td>significantly_better_than_predicted</td>\n",
       "      <td>slight_up_trend</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             ISRC TRANSACTION_COUNTRY_CODE  avg_streams_train  \\\n",
       "609  INH102308037                       GB         108.165414   \n",
       "\n",
       "     avg_streams_test  sum_forecast_errors  \\\n",
       "609         46.285714         1.965631e+18   \n",
       "\n",
       "                 performance_test_period slope_test_period  \n",
       "609  significantly_better_than_predicted   slight_up_trend  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['performance_test_period']=='significantly_better_than_predicted']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8aed5f8b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ece5b43",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57f80c86",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ac378fa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b3c7610",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "42f8612e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import boto3 \n",
    "import sys\n",
    "\n",
    "s3 = boto3.resource('s3')\n",
    "bucket_name = 'dev-cucumbers'\n",
    "bucket = s3.Bucket(bucket_name)\n",
    "\n",
    "def merge_tables_from_s3(folder, prefix, run_id):\n",
    "    ''' Merge all csv files into a single one'''\n",
    "    # Listing all tables for the given run_id\n",
    "    table_names = []\n",
    "    for obj in bucket.objects.filter(Prefix=folder):\n",
    "        table_name = obj.key.split('/')[-1]\n",
    "        prefix_run_id = \"{}_run_{}\".format(prefix, run_id)\n",
    "        if table_name.startswith(prefix_run_id):\n",
    "            table_names.append(table_name)\n",
    "    # Merging all of them in a single dataframe\n",
    "    merged_df = pd.DataFrame() \n",
    "    for table_name in table_names:\n",
    "        obj = bucket.Object(f'{folder}/{table_name}')\n",
    "        table_data = pd.read_csv(obj.get()['Body'])\n",
    "        merged_df = pd.concat([merged_df,table_data])\n",
    "    return merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "20ff4334",
   "metadata": {},
   "outputs": [],
   "source": [
    "# arima_cross_run_20231230132100_chunk_0_par_0.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ec3f776d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_tables_from_s3('path/to/folder3', 'arima_cross', '20231230132100')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "f6e533cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pre_arima_run_123456789012_pre_arima_chunk_0.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_1.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_2.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_3.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_4.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_5.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_6.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_7.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_8.csv',\n",
       " 'pre_arima_run_123456789012_pre_arima_chunk_9.csv']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3 = boto3.resource('s3')\n",
    "bucket_name = 'dev-cucumbers'\n",
    "bucket = s3.Bucket(bucket_name)\n",
    "\n",
    "table_names = []\n",
    "folder ='path/to/folder2'\n",
    "prefix = 'pre_arima'\n",
    "run_id ='123456789012'\n",
    "for obj in bucket.objects.filter(Prefix=folder):\n",
    "    table_name = obj.key.split('/')[-1]\n",
    "    prefix_run_id = \"{}_run_{}\".format(prefix, run_id)\n",
    "    if table_name.startswith(prefix_run_id):\n",
    "        table_names.append(table_name)\n",
    "table_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "03f5238d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACTIVITY_DATE</th>\n",
       "      <th>STREAMS</th>\n",
       "      <th>Fourier</th>\n",
       "      <th>Inflection_Point</th>\n",
       "      <th>ISRC</th>\n",
       "      <th>Fourier_real_part</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-08-15</td>\n",
       "      <td>268</td>\n",
       "      <td>(338.4782723124903+0j)</td>\n",
       "      <td>0</td>\n",
       "      <td>AUI442200152</td>\n",
       "      <td>338.478272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023-08-16</td>\n",
       "      <td>270</td>\n",
       "      <td>(327.80037027288887+0j)</td>\n",
       "      <td>0</td>\n",
       "      <td>AUI442200152</td>\n",
       "      <td>327.800370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023-08-17</td>\n",
       "      <td>291</td>\n",
       "      <td>(317.62747735117705+3.2481953634747436e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>AUI442200152</td>\n",
       "      <td>317.627477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-08-18</td>\n",
       "      <td>272</td>\n",
       "      <td>(308.0016486410911+1.1368683772161602e-14j)</td>\n",
       "      <td>0</td>\n",
       "      <td>AUI442200152</td>\n",
       "      <td>308.001649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-08-19</td>\n",
       "      <td>231</td>\n",
       "      <td>(298.9599348539616+6.496390726949487e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>AUI442200152</td>\n",
       "      <td>298.959935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26035</th>\n",
       "      <td>2023-12-28</td>\n",
       "      <td>220</td>\n",
       "      <td>(243.6143139807787+8.117997781575989e-16j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USQY51353751</td>\n",
       "      <td>243.614314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26036</th>\n",
       "      <td>2023-12-29</td>\n",
       "      <td>254</td>\n",
       "      <td>(243.17833528977232-1.05084058674152e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USQY51353751</td>\n",
       "      <td>243.178335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26037</th>\n",
       "      <td>2023-12-30</td>\n",
       "      <td>261</td>\n",
       "      <td>(242.75083658234635-9.222594085415412e-16j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USQY51353751</td>\n",
       "      <td>242.750837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26038</th>\n",
       "      <td>2023-12-31</td>\n",
       "      <td>362</td>\n",
       "      <td>(242.33544983824322-1.3281136402812983e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USQY51353751</td>\n",
       "      <td>242.335450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26039</th>\n",
       "      <td>2024-01-01</td>\n",
       "      <td>218</td>\n",
       "      <td>(241.9355339713059-8.73926157676731e-16j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USQY51353751</td>\n",
       "      <td>241.935534</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>305340 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ACTIVITY_DATE  STREAMS                                       Fourier  \\\n",
       "0        2023-08-15      268                        (338.4782723124903+0j)   \n",
       "1        2023-08-16      270                       (327.80037027288887+0j)   \n",
       "2        2023-08-17      291  (317.62747735117705+3.2481953634747436e-15j)   \n",
       "3        2023-08-18      272   (308.0016486410911+1.1368683772161602e-14j)   \n",
       "4        2023-08-19      231    (298.9599348539616+6.496390726949487e-15j)   \n",
       "...             ...      ...                                           ...   \n",
       "26035    2023-12-28      220    (243.6143139807787+8.117997781575989e-16j)   \n",
       "26036    2023-12-29      254    (243.17833528977232-1.05084058674152e-15j)   \n",
       "26037    2023-12-30      261   (242.75083658234635-9.222594085415412e-16j)   \n",
       "26038    2023-12-31      362  (242.33544983824322-1.3281136402812983e-15j)   \n",
       "26039    2024-01-01      218     (241.9355339713059-8.73926157676731e-16j)   \n",
       "\n",
       "       Inflection_Point          ISRC  Fourier_real_part  \n",
       "0                     0  AUI442200152         338.478272  \n",
       "1                     0  AUI442200152         327.800370  \n",
       "2                     0  AUI442200152         317.627477  \n",
       "3                     0  AUI442200152         308.001649  \n",
       "4                     0  AUI442200152         298.959935  \n",
       "...                 ...           ...                ...  \n",
       "26035                 0  USQY51353751         243.614314  \n",
       "26036                 0  USQY51353751         243.178335  \n",
       "26037                 0  USQY51353751         242.750837  \n",
       "26038                 0  USQY51353751         242.335450  \n",
       "26039                 0  USQY51353751         241.935534  \n",
       "\n",
       "[305340 rows x 6 columns]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df = pd.DataFrame() \n",
    "\n",
    "for table_name in table_names:\n",
    "    obj = bucket.Object(f'{folder}/{table_name}')\n",
    "    table_data = pd.read_csv(obj.get()['Body'])\n",
    "    merged_df = pd.concat([merged_df,table_data])\n",
    "\n",
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "0f79808a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2181"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "94583963",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fourier_table_run_123456789012_chunk_0.csv',\n",
       " 'fourier_table_run_123456789012_chunk_1.csv',\n",
       " 'fourier_table_run_123456789012_chunk_2.csv',\n",
       " 'fourier_table_run_123456789012_chunk_3.csv',\n",
       " 'fourier_table_run_123456789012_chunk_4.csv',\n",
       " 'fourier_table_run_123456789012_chunk_5.csv',\n",
       " 'fourier_table_run_123456789012_chunk_6.csv',\n",
       " 'fourier_table_run_123456789012_chunk_7.csv',\n",
       " 'fourier_table_run_123456789012_chunk_8.csv',\n",
       " 'fourier_table_run_123456789012_chunk_9.csv']"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_names = []\n",
    "folder ='path/to/folder2'\n",
    "prefix = 'fourier_table'\n",
    "run_id ='123456789012'\n",
    "for obj in bucket.objects.filter(Prefix=folder):\n",
    "    table_name = obj.key.split('/')[-1]\n",
    "    prefix_run_id = \"{}_run_{}\".format(prefix, run_id)\n",
    "    if table_name.startswith(prefix_run_id):\n",
    "        table_names.append(table_name)\n",
    "table_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "0af681a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACTIVITY_DATE</th>\n",
       "      <th>STREAMS</th>\n",
       "      <th>Fourier</th>\n",
       "      <th>Inflection_Point</th>\n",
       "      <th>ISRC</th>\n",
       "      <th>Fourier_real_part</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-08-15</td>\n",
       "      <td>185</td>\n",
       "      <td>(191.82675227538704+0j)</td>\n",
       "      <td>0</td>\n",
       "      <td>ARF412300130</td>\n",
       "      <td>191.826752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023-08-16</td>\n",
       "      <td>190</td>\n",
       "      <td>(195.27978353610737+0j)</td>\n",
       "      <td>0</td>\n",
       "      <td>ARF412300130</td>\n",
       "      <td>195.279784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023-08-17</td>\n",
       "      <td>229</td>\n",
       "      <td>(198.73890217360426+0j)</td>\n",
       "      <td>1</td>\n",
       "      <td>ARF412300130</td>\n",
       "      <td>198.738902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-08-18</td>\n",
       "      <td>256</td>\n",
       "      <td>(202.18464339252168+1.6240976817373718e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>ARF412300130</td>\n",
       "      <td>202.184643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-08-19</td>\n",
       "      <td>274</td>\n",
       "      <td>(205.59795912508085+0j)</td>\n",
       "      <td>0</td>\n",
       "      <td>ARF412300130</td>\n",
       "      <td>205.597959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206215</th>\n",
       "      <td>2023-12-28</td>\n",
       "      <td>416</td>\n",
       "      <td>(438.8948624103172+8.881010641808601e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USLS51713022</td>\n",
       "      <td>438.894862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206216</th>\n",
       "      <td>2023-12-29</td>\n",
       "      <td>352</td>\n",
       "      <td>(439.97201728878866-5.495101975003137e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USLS51713022</td>\n",
       "      <td>439.972017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206217</th>\n",
       "      <td>2023-12-30</td>\n",
       "      <td>397</td>\n",
       "      <td>(441.3417504478791+3.508339767677922e-15j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USLS51713022</td>\n",
       "      <td>441.341750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206218</th>\n",
       "      <td>2023-12-31</td>\n",
       "      <td>339</td>\n",
       "      <td>(442.9785772553104+1.0193891069444909e-14j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USLS51713022</td>\n",
       "      <td>442.978577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206219</th>\n",
       "      <td>2024-01-01</td>\n",
       "      <td>360</td>\n",
       "      <td>(444.8563357825129+1.366650535316288e-14j)</td>\n",
       "      <td>0</td>\n",
       "      <td>USLS51713022</td>\n",
       "      <td>444.856336</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2062200 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ACTIVITY_DATE  STREAMS                                       Fourier  \\\n",
       "0         2023-08-15      185                       (191.82675227538704+0j)   \n",
       "1         2023-08-16      190                       (195.27978353610737+0j)   \n",
       "2         2023-08-17      229                       (198.73890217360426+0j)   \n",
       "3         2023-08-18      256  (202.18464339252168+1.6240976817373718e-15j)   \n",
       "4         2023-08-19      274                       (205.59795912508085+0j)   \n",
       "...              ...      ...                                           ...   \n",
       "206215    2023-12-28      416    (438.8948624103172+8.881010641808601e-15j)   \n",
       "206216    2023-12-29      352   (439.97201728878866-5.495101975003137e-15j)   \n",
       "206217    2023-12-30      397    (441.3417504478791+3.508339767677922e-15j)   \n",
       "206218    2023-12-31      339   (442.9785772553104+1.0193891069444909e-14j)   \n",
       "206219    2024-01-01      360    (444.8563357825129+1.366650535316288e-14j)   \n",
       "\n",
       "        Inflection_Point          ISRC  Fourier_real_part  \n",
       "0                      0  ARF412300130         191.826752  \n",
       "1                      0  ARF412300130         195.279784  \n",
       "2                      1  ARF412300130         198.738902  \n",
       "3                      0  ARF412300130         202.184643  \n",
       "4                      0  ARF412300130         205.597959  \n",
       "...                  ...           ...                ...  \n",
       "206215                 0  USLS51713022         438.894862  \n",
       "206216                 0  USLS51713022         439.972017  \n",
       "206217                 0  USLS51713022         441.341750  \n",
       "206218                 0  USLS51713022         442.978577  \n",
       "206219                 0  USLS51713022         444.856336  \n",
       "\n",
       "[2062200 rows x 6 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df2 = pd.DataFrame() \n",
    "\n",
    "for table_name in table_names:\n",
    "    obj = bucket.Object(f'{folder}/{table_name}')\n",
    "    table_data = pd.read_csv(obj.get()['Body'])\n",
    "    merged_df2 = pd.concat([merged_df2,table_data])\n",
    "\n",
    "merged_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "b7d4f390",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14730"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df2['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "b341eb86",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, date, timedelta\n",
    "\n",
    "def take_isrcs_with_inflection_last_week(fourier_df):\n",
    "    \n",
    "    ''' Function that selects only ISRCs that recorded an inflection point in the last 7 days of their lifecycle '''\n",
    "    \n",
    "    first_day_pred = fourier_df['ACTIVITY_DATE'].max() - timedelta(days=7)\n",
    "    list_for_pred = fourier_df[(fourier_df['Inflection_Point']==1) & \n",
    "                                             (fourier_df['ACTIVITY_DATE'] > first_day_pred)]['ISRC'].unique().tolist()\n",
    "    subset_for_pred = fourier_df[fourier_df['ISRC'].isin(list_for_pred)].copy()\n",
    "    return subset_for_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "fcc9daee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2181"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df2['ACTIVITY_DATE'] = pd.to_datetime(merged_df2['ACTIVITY_DATE'])\n",
    "merged_df2['ACTIVITY_DATE'] = merged_df2['ACTIVITY_DATE'].dt.date\n",
    "\n",
    "subset = take_isrcs_with_inflection_last_week(merged_df2)\n",
    "subset['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "6421fca9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['MOMENTS_chunk_source_123456789012_0_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_1_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_2_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_3_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_4_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_5_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_6_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_7_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_8_10.csv',\n",
       " 'MOMENTS_chunk_source_123456789012_9_10.csv']"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_names = []\n",
    "folder ='path/to/folder2'\n",
    "prefix = 'MOMENTS_chunk_source'\n",
    "run_id ='123456789012'\n",
    "for obj in bucket.objects.filter(Prefix=folder):\n",
    "    table_name = obj.key.split('/')[-1]\n",
    "    prefix_run_id = \"{}_{}\".format(prefix, run_id)\n",
    "    if table_name.startswith(prefix_run_id):\n",
    "        table_names.append(table_name)\n",
    "table_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "c2519021",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>ACTIVITY_DATE</th>\n",
       "      <th>TRACK_NAME</th>\n",
       "      <th>RELEASE_DATE</th>\n",
       "      <th>ARTIST_NAME</th>\n",
       "      <th>ARTIST_ID</th>\n",
       "      <th>STREAMS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ARF412300130</td>\n",
       "      <td>2023-09-22</td>\n",
       "      <td>What a Difference a Day Makes</td>\n",
       "      <td>2023-04-21</td>\n",
       "      <td>The Cooltrane Quartet</td>\n",
       "      <td>645349</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ATN261878503</td>\n",
       "      <td>2023-10-06</td>\n",
       "      <td>Holding My Breath</td>\n",
       "      <td>2018-03-16</td>\n",
       "      <td>Alien Weaponry</td>\n",
       "      <td>956427</td>\n",
       "      <td>228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AUI442200152</td>\n",
       "      <td>2023-09-29</td>\n",
       "      <td>Drinks And Cigars</td>\n",
       "      <td>2022-10-14</td>\n",
       "      <td>Ocean Alley</td>\n",
       "      <td>951854</td>\n",
       "      <td>257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AUAB01800693</td>\n",
       "      <td>2023-11-22</td>\n",
       "      <td>The Lion and the Unicorn</td>\n",
       "      <td>2018-03-02</td>\n",
       "      <td>The Wiggles</td>\n",
       "      <td>2508923</td>\n",
       "      <td>309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AUI442200152</td>\n",
       "      <td>2023-09-19</td>\n",
       "      <td>Drinks And Cigars</td>\n",
       "      <td>2022-10-14</td>\n",
       "      <td>Ocean Alley</td>\n",
       "      <td>951854</td>\n",
       "      <td>285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206215</th>\n",
       "      <td>USLS52336811</td>\n",
       "      <td>2023-10-13</td>\n",
       "      <td>Makin' My Way</td>\n",
       "      <td>2023-06-02</td>\n",
       "      <td>Neal Acree</td>\n",
       "      <td>2587416</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206216</th>\n",
       "      <td>USLS52336811</td>\n",
       "      <td>2023-09-18</td>\n",
       "      <td>Makin' My Way</td>\n",
       "      <td>2023-06-02</td>\n",
       "      <td>Neal Acree</td>\n",
       "      <td>2587416</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206217</th>\n",
       "      <td>USP6L2100631</td>\n",
       "      <td>2023-10-17</td>\n",
       "      <td>Fck Boys</td>\n",
       "      <td>2021-03-12</td>\n",
       "      <td>Blxst</td>\n",
       "      <td>1960782</td>\n",
       "      <td>968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206218</th>\n",
       "      <td>USLS51015212</td>\n",
       "      <td>2023-09-01</td>\n",
       "      <td>Flow Like Water</td>\n",
       "      <td>2016-08-19</td>\n",
       "      <td>James Newton Howard</td>\n",
       "      <td>701806</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206219</th>\n",
       "      <td>USRE11300312</td>\n",
       "      <td>2023-12-29</td>\n",
       "      <td>Before You Accuse Me</td>\n",
       "      <td>1992-08-25</td>\n",
       "      <td>Eric Clapton</td>\n",
       "      <td>2905391</td>\n",
       "      <td>253</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2062200 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                ISRC ACTIVITY_DATE                     TRACK_NAME  \\\n",
       "0       ARF412300130    2023-09-22  What a Difference a Day Makes   \n",
       "1       ATN261878503    2023-10-06              Holding My Breath   \n",
       "2       AUI442200152    2023-09-29              Drinks And Cigars   \n",
       "3       AUAB01800693    2023-11-22       The Lion and the Unicorn   \n",
       "4       AUI442200152    2023-09-19              Drinks And Cigars   \n",
       "...              ...           ...                            ...   \n",
       "206215  USLS52336811    2023-10-13                  Makin' My Way   \n",
       "206216  USLS52336811    2023-09-18                  Makin' My Way   \n",
       "206217  USP6L2100631    2023-10-17                       Fck Boys   \n",
       "206218  USLS51015212    2023-09-01                Flow Like Water   \n",
       "206219  USRE11300312    2023-12-29           Before You Accuse Me   \n",
       "\n",
       "       RELEASE_DATE            ARTIST_NAME  ARTIST_ID  STREAMS  \n",
       "0        2023-04-21  The Cooltrane Quartet     645349      267  \n",
       "1        2018-03-16         Alien Weaponry     956427      228  \n",
       "2        2022-10-14            Ocean Alley     951854      257  \n",
       "3        2018-03-02            The Wiggles    2508923      309  \n",
       "4        2022-10-14            Ocean Alley     951854      285  \n",
       "...             ...                    ...        ...      ...  \n",
       "206215   2023-06-02             Neal Acree    2587416      149  \n",
       "206216   2023-06-02             Neal Acree    2587416      171  \n",
       "206217   2021-03-12                  Blxst    1960782      968  \n",
       "206218   2016-08-19    James Newton Howard     701806      103  \n",
       "206219   1992-08-25           Eric Clapton    2905391      253  \n",
       "\n",
       "[2062200 rows x 7 columns]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source = pd.DataFrame() \n",
    "\n",
    "for table_name in table_names:\n",
    "    obj = bucket.Object(f'{folder}/{table_name}')\n",
    "    table_data = pd.read_csv(obj.get()['Body'])\n",
    "    source = pd.concat([source,table_data])\n",
    "\n",
    "source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "dd678f10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14730"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "3f74afb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['arima_cross_run_test_run_for_Rain_chunk_0_par_0.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_1.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_10.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_11.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_12.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_13.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_14.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_15.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_16.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_17.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_18.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_19.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_2.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_20.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_21.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_22.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_23.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_24.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_25.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_26.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_27.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_28.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_29.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_3.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_30.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_31.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_32.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_33.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_34.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_35.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_36.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_37.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_38.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_39.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_4.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_40.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_41.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_42.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_43.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_44.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_45.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_46.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_47.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_48.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_49.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_5.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_50.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_51.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_52.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_53.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_6.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_7.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_8.csv',\n",
       " 'arima_cross_run_test_run_for_Rain_chunk_0_par_9.csv']"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_names = []\n",
    "folder ='eimpara/pipelines/moments'\n",
    "prefix = 'arima_cross'\n",
    "run_id ='test_run_for_Rain'\n",
    "for obj in bucket.objects.filter(Prefix=folder):\n",
    "    table_name = obj.key.split('/')[-1]\n",
    "    prefix_run_id = \"{}_run_{}\".format(prefix, run_id)\n",
    "    if table_name.startswith(prefix_run_id):\n",
    "        table_names.append(table_name)\n",
    "table_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "599ed99f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>pred_type</th>\n",
       "      <th>mse</th>\n",
       "      <th>avg_streams_train</th>\n",
       "      <th>avg_streams_test</th>\n",
       "      <th>median_streams_train</th>\n",
       "      <th>median_streams_test</th>\n",
       "      <th>linear_gradient_train</th>\n",
       "      <th>linear_gradient_test</th>\n",
       "      <th>sum_forecast_errors</th>\n",
       "      <th>len_df</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>170158.631936</td>\n",
       "      <td>3595.007519</td>\n",
       "      <td>3426.000000</td>\n",
       "      <td>4017.0</td>\n",
       "      <td>3530.0</td>\n",
       "      <td>-15.842105</td>\n",
       "      <td>-5.857143</td>\n",
       "      <td>2151.331</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AEA2D2300306</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>329.247662</td>\n",
       "      <td>109.977444</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>111.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>-1.045113</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>27.629</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AEA2W2369250</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>2069.608227</td>\n",
       "      <td>150.736842</td>\n",
       "      <td>119.142857</td>\n",
       "      <td>124.0</td>\n",
       "      <td>113.0</td>\n",
       "      <td>-1.097744</td>\n",
       "      <td>1.142857</td>\n",
       "      <td>-79.177</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ARA340900073</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>45626.714556</td>\n",
       "      <td>740.278195</td>\n",
       "      <td>609.571429</td>\n",
       "      <td>689.0</td>\n",
       "      <td>536.0</td>\n",
       "      <td>-2.511278</td>\n",
       "      <td>2.142857</td>\n",
       "      <td>997.238</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ARBAA1600002</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>7731.004734</td>\n",
       "      <td>2360.887218</td>\n",
       "      <td>2113.000000</td>\n",
       "      <td>2356.0</td>\n",
       "      <td>2123.0</td>\n",
       "      <td>-1.774436</td>\n",
       "      <td>22.571429</td>\n",
       "      <td>291.402</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ATN262206005</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>1453.118431</td>\n",
       "      <td>393.443609</td>\n",
       "      <td>227.714286</td>\n",
       "      <td>374.0</td>\n",
       "      <td>236.0</td>\n",
       "      <td>-1.781955</td>\n",
       "      <td>11.571429</td>\n",
       "      <td>-180.212</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ATN262206006</td>\n",
       "      <td>actuals_below_predicted</td>\n",
       "      <td>3114.489249</td>\n",
       "      <td>321.706767</td>\n",
       "      <td>195.285714</td>\n",
       "      <td>322.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>-2.631579</td>\n",
       "      <td>-3.428571</td>\n",
       "      <td>-326.185</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ATN262210902</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>2501.271021</td>\n",
       "      <td>208.353383</td>\n",
       "      <td>192.714286</td>\n",
       "      <td>205.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>-0.736842</td>\n",
       "      <td>-6.857143</td>\n",
       "      <td>-221.059</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ATN262210904</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>22070.062652</td>\n",
       "      <td>760.676692</td>\n",
       "      <td>678.571429</td>\n",
       "      <td>760.0</td>\n",
       "      <td>673.0</td>\n",
       "      <td>-2.300752</td>\n",
       "      <td>1.285714</td>\n",
       "      <td>-695.172</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>actuals_crossing_predicted</td>\n",
       "      <td>1024.698785</td>\n",
       "      <td>218.639098</td>\n",
       "      <td>200.714286</td>\n",
       "      <td>213.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>-0.691729</td>\n",
       "      <td>-0.428571</td>\n",
       "      <td>44.450</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>213 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ISRC                   pred_type            mse  \\\n",
       "0   AEA0Q2149133  actuals_crossing_predicted  170158.631936   \n",
       "1   AEA2D2300306  actuals_crossing_predicted     329.247662   \n",
       "2   AEA2W2369250  actuals_crossing_predicted    2069.608227   \n",
       "3   ARA340900073  actuals_crossing_predicted   45626.714556   \n",
       "0   ARBAA1600002  actuals_crossing_predicted    7731.004734   \n",
       "..           ...                         ...            ...   \n",
       "3   ATN262206005  actuals_crossing_predicted    1453.118431   \n",
       "0   ATN262206006     actuals_below_predicted    3114.489249   \n",
       "1   ATN262210902  actuals_crossing_predicted    2501.271021   \n",
       "2   ATN262210904  actuals_crossing_predicted   22070.062652   \n",
       "3   ATN262210905  actuals_crossing_predicted    1024.698785   \n",
       "\n",
       "    avg_streams_train  avg_streams_test  median_streams_train  \\\n",
       "0         3595.007519       3426.000000                4017.0   \n",
       "1          109.977444         61.000000                 111.0   \n",
       "2          150.736842        119.142857                 124.0   \n",
       "3          740.278195        609.571429                 689.0   \n",
       "0         2360.887218       2113.000000                2356.0   \n",
       "..                ...               ...                   ...   \n",
       "3          393.443609        227.714286                 374.0   \n",
       "0          321.706767        195.285714                 322.0   \n",
       "1          208.353383        192.714286                 205.0   \n",
       "2          760.676692        678.571429                 760.0   \n",
       "3          218.639098        200.714286                 213.0   \n",
       "\n",
       "    median_streams_test  linear_gradient_train  linear_gradient_test  \\\n",
       "0                3530.0             -15.842105             -5.857143   \n",
       "1                  61.0              -1.045113              0.285714   \n",
       "2                 113.0              -1.097744              1.142857   \n",
       "3                 536.0              -2.511278              2.142857   \n",
       "0                2123.0              -1.774436             22.571429   \n",
       "..                  ...                    ...                   ...   \n",
       "3                 236.0              -1.781955             11.571429   \n",
       "0                 204.0              -2.631579             -3.428571   \n",
       "1                 201.0              -0.736842             -6.857143   \n",
       "2                 673.0              -2.300752              1.285714   \n",
       "3                 200.0              -0.691729             -0.428571   \n",
       "\n",
       "    sum_forecast_errors  len_df  \n",
       "0              2151.331     140  \n",
       "1                27.629     140  \n",
       "2               -79.177     140  \n",
       "3               997.238     140  \n",
       "0               291.402     140  \n",
       "..                  ...     ...  \n",
       "3              -180.212     140  \n",
       "0              -326.185     140  \n",
       "1              -221.059     140  \n",
       "2              -695.172     140  \n",
       "3                44.450     140  \n",
       "\n",
       "[213 rows x 11 columns]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arima_cross = pd.DataFrame() \n",
    "\n",
    "for table_name in table_names:\n",
    "    obj = bucket.Object(f'{folder}/{table_name}')\n",
    "    table_data = pd.read_csv(obj.get()['Body'])\n",
    "    arima_cross = pd.concat([arima_cross,table_data])\n",
    "\n",
    "arima_cross"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "5f8807c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "213"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arima_cross['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "1f73c95a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['arima_timeseries_run_test_run_for_Rain_chunk_0_par_0.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_1.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_10.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_11.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_12.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_13.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_14.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_15.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_16.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_17.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_18.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_19.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_2.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_20.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_21.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_22.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_23.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_24.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_25.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_26.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_27.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_28.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_29.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_3.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_30.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_31.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_32.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_33.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_34.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_35.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_36.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_37.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_38.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_39.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_4.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_40.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_41.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_42.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_43.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_44.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_45.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_46.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_47.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_48.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_49.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_5.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_50.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_51.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_52.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_53.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_6.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_7.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_8.csv',\n",
       " 'arima_timeseries_run_test_run_for_Rain_chunk_0_par_9.csv']"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_names = []\n",
    "folder ='eimpara/pipelines/moments'\n",
    "prefix = 'arima_timeseries'\n",
    "run_id ='test_run_for_Rain'\n",
    "for obj in bucket.objects.filter(Prefix=folder):\n",
    "    table_name = obj.key.split('/')[-1]\n",
    "    prefix_run_id = \"{}_run_{}\".format(prefix, run_id)\n",
    "    if table_name.startswith(prefix_run_id):\n",
    "        table_names.append(table_name)\n",
    "table_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "833d2a66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISRC</th>\n",
       "      <th>ACTIVITY_DATE</th>\n",
       "      <th>STREAMS</th>\n",
       "      <th>predicted_values</th>\n",
       "      <th>len_df</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>2023-08-15</td>\n",
       "      <td>5035</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>2023-08-16</td>\n",
       "      <td>4677</td>\n",
       "      <td>2396.434000</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>2023-08-17</td>\n",
       "      <td>3207</td>\n",
       "      <td>3604.420579</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>2023-08-18</td>\n",
       "      <td>1876</td>\n",
       "      <td>3194.587746</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AEA0Q2149133</td>\n",
       "      <td>2023-08-19</td>\n",
       "      <td>1749</td>\n",
       "      <td>2253.813742</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>555</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>2023-12-28</td>\n",
       "      <td>220</td>\n",
       "      <td>203.187480</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>2023-12-29</td>\n",
       "      <td>213</td>\n",
       "      <td>229.397440</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>2023-12-30</td>\n",
       "      <td>199</td>\n",
       "      <td>191.136323</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>2023-12-31</td>\n",
       "      <td>228</td>\n",
       "      <td>153.911597</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>ATN262210905</td>\n",
       "      <td>2024-01-01</td>\n",
       "      <td>171</td>\n",
       "      <td>200.390452</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29820 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             ISRC ACTIVITY_DATE  STREAMS  predicted_values  len_df\n",
       "0    AEA0Q2149133    2023-08-15     5035          0.000000     140\n",
       "1    AEA0Q2149133    2023-08-16     4677       2396.434000     140\n",
       "2    AEA0Q2149133    2023-08-17     3207       3604.420579     140\n",
       "3    AEA0Q2149133    2023-08-18     1876       3194.587746     140\n",
       "4    AEA0Q2149133    2023-08-19     1749       2253.813742     140\n",
       "..            ...           ...      ...               ...     ...\n",
       "555  ATN262210905    2023-12-28      220        203.187480     140\n",
       "556  ATN262210905    2023-12-29      213        229.397440     140\n",
       "557  ATN262210905    2023-12-30      199        191.136323     140\n",
       "558  ATN262210905    2023-12-31      228        153.911597     140\n",
       "559  ATN262210905    2024-01-01      171        200.390452     140\n",
       "\n",
       "[29820 rows x 5 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arima_timeseries = pd.DataFrame() \n",
    "\n",
    "for table_name in table_names:\n",
    "    obj = bucket.Object(f'{folder}/{table_name}')\n",
    "    table_data = pd.read_csv(obj.get()['Body'])\n",
    "    arima_timeseries = pd.concat([arima_timeseries,table_data])\n",
    "\n",
    "arima_timeseries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "c5beca00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "213"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arima_timeseries['ISRC'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "479f85a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2904f9ed",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}