import boto3 import pandas as pd def merge_tables_from_s3(path, prefix, run_id): ''' Merge all csv files into a single one''' # Listing all tables for the given run_id table_names = [] for obj in bucket.objects.filter(Prefix=path): table_name = obj.key.split('/')[-1] prefix_run_id = "{}_{}".format(prefix, run_id) if table_name.startswith(prefix_run_id): table_names.append(table_name) # Merging all of them in a single dataframe merged_df = pd.DataFrame() for table_name in table_names: obj = bucket.Object(f'{path}/{table_name}') table_data = pd.read_csv(obj.get()['Body']) merged_df = pd.concat([merged_df,table_data]) return merged_df