{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bbfa9fa9-7275-4a00-ad8c-a156298b18e9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import boto3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e53e9553-fb5e-478f-b4fb-5b9db154a703",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.resource('s3')\n",
    "bucket_name = 'dev-cucumbers'\n",
    "bucket = s3.Bucket(bucket_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "79cfbf19",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'eimpara/ARIMA'\n",
    "table_names = []\n",
    "\n",
    "for obj in bucket.objects.filter(Prefix=path):\n",
    "    table_name = obj.key.split('/')[-1]\n",
    "    if len(table_name) > 0:\n",
    "        table_names.append(table_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "77a40409",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1692958078682143.2_0.csv',\n",
       " '1692958078682143.2_1.csv',\n",
       " '1692958078682143.2_2.csv',\n",
       " '1692958078682143.2_3.csv',\n",
       " '1692958078682143.2_4.csv',\n",
       " '1692958078682143.2_5.csv',\n",
       " '1692958078682143.2_6.csv',\n",
       " '1692958078682143.2_7.csv',\n",
       " '1692958078682143.2_8.csv',\n",
       " '1692958078682143.2_9.csv']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8b1d4caa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n",
      "/tmp/ipykernel_29970/1951938181.py:6: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  merged_df = merged_df.append(table_data)\n"
     ]
    }
   ],
   "source": [
    "merged_df = pd.DataFrame() \n",
    "\n",
    "for table_name in table_names:\n",
    "    obj = bucket.Object(f'{path}/{table_name}')\n",
    "    table_data = pd.read_csv(obj.get()['Body'])\n",
    "    merged_df = merged_df.append(table_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "34637e13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(219, 11)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a219581d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_dataframe_s3(df):\n",
    "    s3 = boto3.client('s3')\n",
    "    bucket_name = 'dev-cucumbers'\n",
    "    filepath = \"eimpara/ARIMA/Positive_Sign_table_for_clustering_25Aug23.csv\"\n",
    "    csv_buffer = df.to_csv(index=False).encode('utf-8')\n",
    "    # Save the CSV file to S3\n",
    "    s3.put_object(Body=csv_buffer, Bucket=bucket_name, Key=filepath)\n",
    "    print(f\"Table saved to S3 bucket: {bucket_name}, with file name: {filepath}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "758c2b1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dataframe_s3(merged_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}