import csv from datetime import datetime from io import StringIO import logging from typing import Any, Dict, List log = logging.getLogger(__name__) def generate_random_filename(filename_prefix: str) -> str: """Generates a random filename with a given prefix and current datetime.""" now = datetime.now() timestamp = now.strftime('%Y%m%d%H%M%S') generated_filename = f'{filename_prefix}IntegrationTest_{timestamp}.csv' return generated_filename def create_csv_buffer(data: List[Dict[str, Any]]) -> StringIO: """Creates an in-memory CSV buffer from the provided data.""" fieldnames = data[0].keys() csv_buffer = StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) writer.writeheader() writer.writerows(data) return csv_buffer def read_csv_from_buffer(buffer: StringIO) -> List[Dict[str, str]]: """ Reads CSV content from a StringIO buffer and returns it as a list of dictionaries. """ buffer.seek(0) # Ensure we're reading from the beginning reader = csv.DictReader(buffer) return [row for row in reader] def create_and_upload_csv_to_s3( s3_client: Any, file_data: List[Dict[str, Any]], filename_prefix: str, s3_folder: str, bucket: str, ) -> tuple[StringIO, str]: """ Creates a CSV file in memory, uploads it to S3, and returns the buffer and S3 key. """ local_filename = generate_random_filename(filename_prefix) s3_key = f'{s3_folder}/{local_filename}' csv_buffer = create_csv_buffer(file_data) s3_client.put(bucket, s3_key, csv_buffer.getvalue().encode('utf-8')) log.info(f'Uploaded {local_filename} to S3 bucket {bucket}') return csv_buffer, s3_key