{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from botocore.client import Config\n",
    "\n",
    "# Get the service client.\n",
    "s3 = boto3.client('s3', config=Config(signature_version='s3v4'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "objs = s3.list_objects_v2(Bucket='wl-rds-exports', Prefix='')['Contents']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = [o['Key'] for o in objs if '.gz' in o['Key']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scgzip/2021-04-07/sc_track_stats_58.tsv.gz',\n",
       " 'scgzip/2021-04-07/sc_track_stats_59.tsv.gz',\n",
       " 'scgzip/2021-04-07/sc_track_stats_60.tsv.gz',\n",
       " 'scgzip/2021-04-07/sc_track_stats_61.tsv.gz',\n",
       " 'scgzip/2021-04-07/sc_track_stats_62.tsv.gz',\n",
       " 'scgzip/2021-04-07/sc_track_stats_63.tsv.gz',\n",
       " 'scgzip/2021-04-07/sc_track_stats_64.tsv.gz',\n",
       " 'scgzip/2021-04-07/sc_track_stats_65.tsv.gz']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_url(filename):\n",
    "    url = s3.generate_presigned_url(\n",
    "        ClientMethod='get_object',\n",
    "        Params={\n",
    "            'Bucket': 'wl-rds-exports',\n",
    "            'Key': filename,\n",
    "        },\n",
    "        ExpiresIn=5*24*60*60  # seconds\n",
    "    )\n",
    "    return url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from importlib import reload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'boto3' from '/Users/joel/src/thundr/tracker/venv/lib/python3.7/site-packages/boto3/__init__.py'>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(boto3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_urls = [(f.split('/')[-1], make_url(f)) for f in filenames]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sc_track_stats_58.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_58.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=4b7039c8baf1d2ac5e97fceb351b402746f73c7cb23c95aad0c1fce515e3eca8\n",
      "\n",
      "sc_track_stats_59.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_59.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=6b74bca62a88679fcec898115600c4a895d33353a32559d117a82ed5d967211e\n",
      "\n",
      "sc_track_stats_60.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_60.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=2af2c908567f1289e2968116c7fda0061009980640599125ce17b582127552cb\n",
      "\n",
      "sc_track_stats_61.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_61.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=a019dfac2b733fdfcb1b0f11c54bb4090baf7d6adddd4359c36dc2c21d5b9314\n",
      "\n",
      "sc_track_stats_62.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_62.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=d802c402205d080b1d820e5166874812159b7108f41e9c29d98ae6e2295a9ea5\n",
      "\n",
      "sc_track_stats_63.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_63.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=3a204a460bd14012fa1a47504aa6c113e86e56202cbd925e878323eba6f0d39b\n",
      "\n",
      "sc_track_stats_64.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_64.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=d0f8fe074821e0cca8725e668ed8d00145d45339ade631ffacd6f992d63af1d2\n",
      "\n",
      "sc_track_stats_65.tsv.gz:\n",
      "https://wl-rds-exports.s3.amazonaws.com/scgzip/2021-04-07/sc_track_stats_65.tsv.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAUWIR57QNJ4VRX4GZ%2F20210426%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20210426T155036Z&X-Amz-Expires=432000&X-Amz-SignedHeaders=host&X-Amz-Security-Token=FwoGZXIvYXdzEMb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaDCgYcjKBahNKTxSpmiKGAa4JdYkWbqyvtYCsUVQZr1oGHL9nifUU0W2NhKCyBQUWvEeZ2Gw2ZVQ%2F4xBISWkqGcDHYhVfvA8WrjUBN9k%2Bqa4a4G%2Bn3vnEd4UY6xLFAh%2FmA1BN0Yvxc8S%2Bq3AWUXCOlcO9qAdlq5lxmIpqCSp2PC3XcRvwF3l4fM7f3VTV3MbZkxutD6sNKMDvmoQGMigeQWFZ1kvYtWG9izO3w61F0ZQm6fibbZxZjZBtGhKuT7iU%2FEfRiZqw&X-Amz-Signature=31f57d18e606c98cb00d45b728b35c511264b4f3b40c26462f14c16609479958\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(''.join([f\"{name}:\\n{url}\\n\\n\" for name, url in all_urls]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}