{
"cells": [
{
"cell_type": "markdown",
"id": "c675fa47-cc91-47d7-b9a3-88b0dab19ca0",
"metadata": {},
"source": [
"https://hands-on.cloud/boto3-secrets-manager-tutorial/?utm_content=cmp-true"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ff315bff-4fe3-4160-9d7c-3f36baae056a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import boto3\n",
"client = boto3.client('secretsmanager')\n",
"response = client.list_secrets()\n",
"print(response['SecretList'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f21f7d53-6bf9-433f-98d1-5175987a5a8a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import boto3 \n",
"import json \n",
"client = boto3.client('secretsmanager')\n",
"response = client.get_secret_value(\n",
" SecretId=sec_id\n",
")\n",
"database_secrets = json.loads(response['SecretString'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d6198783-115a-4989-9ad6-d7906c1d5645",
"metadata": {},
"outputs": [],
"source": [
"!pip install \"snowflake-connector-python[pandas]\""
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "198896a3-e66e-4b7d-90cd-05df937e1dda",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import boto3\n",
"import pandas as pd\n",
"import snowflake\n",
"import snowflake.connector"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "97003d91-29ed-43cd-9b6f-12c812460fc8",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"'Xm2cdZQZB7hi3tCglT7h'"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sec_id = 'dev/streamlit-audience-etl/SNOWFLAKE_PASSWORD'\n",
"\n",
"\n",
"def get_secret_value(name, version=None):\n",
" \"\"\"Gets the value of a secret from aws.\n",
"\n",
" Version (if defined) is used to retrieve a particular version of\n",
" the secret.\n",
"\n",
" \"\"\"\n",
" secrets_client = boto3.client(\"secretsmanager\")\n",
" kwargs = {'SecretId': name}\n",
" if version is not None:\n",
" kwargs['VersionStage'] = version\n",
" response = secrets_client.get_secret_value(**kwargs)\n",
" return response\n",
"\n",
"pwd = get_secret_value(sec_id)['SecretString']\n",
"pwd"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "21b36d93-1e6a-4209-9d96-ab745e8bb75b",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"SNOWFLAKE_ACCOUNT = 'orchard'\n",
"SNOWFLAKE_DATABASE = 'DEV_ENGINEERING'\n",
"SNOWFLAKE_SCHEMA = 'RBOMBERG_DBT'\n",
"SNOWFLAKE_WAREHOUSE = \"DEV_OWS_WAREHOUSE\"\n",
"SNOWFLAKE_ROLE = \"DEV_STREAMLIT_AUDIENCE_ETL_ROLE\"\n",
"SNOWFLAKE_USER = \"DEV_STREAMLIT_AUDIENCE_ETL\"\n",
"SNOWFLAKE_PASSWORD = pwd"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "3e52e91e-8b08-4b43-ae19-17c304b2b1f2",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ctx = snowflake.connector.connect(\n",
" account=SNOWFLAKE_ACCOUNT,\n",
" user=SNOWFLAKE_USER,\n",
" password=SNOWFLAKE_PASSWORD,\n",
" role=SNOWFLAKE_ROLE,\n",
" warehouse=SNOWFLAKE_WAREHOUSE,\n",
" database=SNOWFLAKE_DATABASE,\n",
" schema=SNOWFLAKE_SCHEMA,\n",
")\n",
"\n",
"cur = ctx.cursor()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "93cc89ce-15c9-4d92-ac84-aef62714e937",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"\n",
"with open('test.sql') as f:\n",
" query = f.read()\n",
"f.close()\n",
"\n",
"cur.execute(query)\n",
"dataframe = cur.fetch_pandas_all()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "04e587b9-8eb4-4512-8fd5-ecf169d16b02",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(10, 4)\n",
" SPOTIFY_ARTIST_ID CM_TAG TAG_NAME TAG_SOURCE\n",
"0 15qI5w4XJFLRMwOp2VrlD5 405147 pop everynoise\n",
"1 2ajhZ7EA6Dec0kaWiKCApF 405147 pop everynoise\n",
"2 4tqPTKknmeQK1uDEFVfX4G 405147 pop everynoise\n",
"3 3dH7pcBScIJQboDyMzUzez 405443 rai everynoise\n",
"4 5EvFsr3kj42KNv97ZEnqij 405443 rai everynoise\n"
]
}
],
"source": [
"print(dataframe.shape)\n",
"print(dataframe.head())"
]
},
{
"cell_type": "markdown",
"id": "b0e18b42-8976-468f-bdc8-08176c131561",
"metadata": {},
"source": [
"Bucket access"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "26aebf69-1b34-4e5d-8085-3b0e920eec6b",
"metadata": {
"tags": []
},
"outputs": [
{
"ename": "ClientError",
"evalue": "An error occurred (AccessDenied) when calling the ListObjects operation: Access Denied",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mClientError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[2], line 9\u001b[0m\n\u001b[1;32m 7\u001b[0m my_bucket \u001b[38;5;241m=\u001b[39m s3\u001b[38;5;241m.\u001b[39mBucket(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdev-cucumbers\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 8\u001b[0m my_bucket \u001b[38;5;241m=\u001b[39m s3\u001b[38;5;241m.\u001b[39mBucket(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstreamlit-audience\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 9\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m my_bucket_object \u001b[38;5;129;01min\u001b[39;00m my_bucket\u001b[38;5;241m.\u001b[39mobjects\u001b[38;5;241m.\u001b[39mall():\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28mprint\u001b[39m(my_bucket_object\u001b[38;5;241m.\u001b[39mkey)\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/boto3/resources/collection.py:81\u001b[0m, in \u001b[0;36mResourceCollection.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 78\u001b[0m limit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_params\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlimit\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m 80\u001b[0m count \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m page \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpages():\n\u001b[1;32m 82\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m page:\n\u001b[1;32m 83\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m item\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/boto3/resources/collection.py:171\u001b[0m, in \u001b[0;36mResourceCollection.pages\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;66;03m# Now that we have a page iterator or single page of results\u001b[39;00m\n\u001b[1;32m 169\u001b[0m \u001b[38;5;66;03m# we start processing and yielding individual items.\u001b[39;00m\n\u001b[1;32m 170\u001b[0m count \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m--> 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m page \u001b[38;5;129;01min\u001b[39;00m pages:\n\u001b[1;32m 172\u001b[0m page_items \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_handler(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parent, params, page):\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/paginate.py:269\u001b[0m, in \u001b[0;36mPageIterator.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 267\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_inject_starting_params(current_kwargs)\n\u001b[1;32m 268\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 269\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcurrent_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 270\u001b[0m parsed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_extract_parsed_response(response)\n\u001b[1;32m 271\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m first_request:\n\u001b[1;32m 272\u001b[0m \u001b[38;5;66;03m# The first request is handled differently. We could\u001b[39;00m\n\u001b[1;32m 273\u001b[0m \u001b[38;5;66;03m# possibly have a resume/starting token that tells us where\u001b[39;00m\n\u001b[1;32m 274\u001b[0m \u001b[38;5;66;03m# to index into the retrieved page.\u001b[39;00m\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/paginate.py:357\u001b[0m, in \u001b[0;36mPageIterator._make_request\u001b[0;34m(self, current_kwargs)\u001b[0m\n\u001b[1;32m 356\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_make_request\u001b[39m(\u001b[38;5;28mself\u001b[39m, current_kwargs):\n\u001b[0;32m--> 357\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_method\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mcurrent_kwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:530\u001b[0m, in \u001b[0;36mClientCreator._create_api_method.._api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 527\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpy_operation_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m() only accepts keyword arguments.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 528\u001b[0m )\n\u001b[1;32m 529\u001b[0m \u001b[38;5;66;03m# The \"self\" in this scope is referring to the BaseClient.\u001b[39;00m\n\u001b[0;32m--> 530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_api_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43moperation_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:964\u001b[0m, in \u001b[0;36mBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 962\u001b[0m error_code \u001b[38;5;241m=\u001b[39m parsed_response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCode\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 963\u001b[0m error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m--> 964\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 966\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parsed_response\n",
"\u001b[0;31mClientError\u001b[0m: An error occurred (AccessDenied) when calling the ListObjects operation: Access Denied"
]
}
],
"source": [
"import boto3\n",
"session = boto3.Session()\n",
"\n",
"#Then use the session to get the resource\n",
"s3 = session.resource('s3')\n",
"\n",
"my_bucket = s3.Bucket('dev-cucumbers')\n",
"my_bucket = s3.Bucket('streamlit-audience')\n",
"for my_bucket_object in my_bucket.objects.all():\n",
" print(my_bucket_object.key)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "df9c3c4c-b27b-4d0b-aa35-64995ecea9dc",
"metadata": {
"tags": []
},
"outputs": [
{
"ename": "ClientError",
"evalue": "An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mClientError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[3], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Retrieve the list of existing buckets\u001b[39;00m\n\u001b[1;32m 2\u001b[0m s3 \u001b[38;5;241m=\u001b[39m boto3\u001b[38;5;241m.\u001b[39mclient(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms3\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43ms3\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlist_buckets\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;66;03m# Output the bucket names\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mExisting buckets:\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:530\u001b[0m, in \u001b[0;36mClientCreator._create_api_method.._api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 527\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpy_operation_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m() only accepts keyword arguments.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 528\u001b[0m )\n\u001b[1;32m 529\u001b[0m \u001b[38;5;66;03m# The \"self\" in this scope is referring to the BaseClient.\u001b[39;00m\n\u001b[0;32m--> 530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_api_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43moperation_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/client.py:964\u001b[0m, in \u001b[0;36mBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 962\u001b[0m error_code \u001b[38;5;241m=\u001b[39m parsed_response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCode\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 963\u001b[0m error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m--> 964\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 966\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parsed_response\n",
"\u001b[0;31mClientError\u001b[0m: An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied"
]
}
],
"source": [
"# Retrieve the list of existing buckets\n",
"s3 = boto3.client('s3')\n",
"response = s3.list_buckets()\n",
"\n",
"# Output the bucket names\n",
"print('Existing buckets:')\n",
"for bucket in response['Buckets']:\n",
" print(f' {bucket[\"Name\"]}')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "conda_python3",
"language": "python",
"name": "conda_python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}