"""Tasks of the GfK Physical Ingestion Workflow.""" import datetime import os import tempfile import zipfile import boto3 from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status import paramiko from feed_ingestion.flows.gfk_physical import config from feed_ingestion.tasks import check_status @task.decorate(timeout=700) def bootstrap(activity, date, reload): """Bootstrap workflow by getting the correct configurations. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str or None): If 'True' reload status. Returns: dict: Context. """ date_obj = ( datetime.datetime.strptime(date, '%Y-%m-%d').date() if date else datetime.date.today() ) date = date_obj.strftime('%Y-%m-%d') if reload == 'True': activity.logger.info( f'Delete status for feed: {config.feed_name} {date}' ) garcon_feed_status.delete_status(config.feed_name, date) elif ( garcon_feed_status.get_overall_status(config.feed_name, date) == garcon_feed_status.STATUS_INGESTED ): activity.logger.info(f'Feed already ingested for {date}') return { 'stop': True, 'message': f'{config.feed_name} is already ingested for {date}', } activity.logger.info(f'Bootstrap flow for {date}') yy = date_obj.strftime('%y') mm = date_obj.strftime('%m') dd = date_obj.strftime('%d') filename = config.file_template.format(yy=yy, mm=mm, dd=dd) archive_dir = config.s3['archive_path'].format(date=date) processed_dir = config.s3['processed_path'].format(date=date) processed_filename = filename.replace('.ZIP', '.csv') s3_dir_path = f's3://{config.data_bucket}/{processed_dir}' return dict( feed_name=config.feed_name, date=date, filename=filename, archive_path=f'{archive_dir}{filename}', processed_path=f'{processed_dir}{processed_filename}', staging_raw_table=config.staging_raw_table, s3_dir_path=s3_dir_path, ) def _get_sftp_connection(sftp_creds): """Establish an SFTP connection using paramiko. Args: sftp_creds (dict): SFTP credentials with keys host, username, password, and optionally port. Returns: paramiko.SFTPClient: Connected SFTP client. """ paramiko.sftp_file.SFTPFile.MAX_REQUEST_SIZE = 4194304 transport = paramiko.Transport( (sftp_creds['host'], sftp_creds.get('port', 22)) ) transport.connect( username=sftp_creds['username'], password=sftp_creds['password'], ) return paramiko.SFTPClient.from_transport(transport) @task.decorate(timeout=1800) @check_status() def fetch_from_sftp( activity, feed_name, date, filename, archive_path, processed_path ): """Download ZIP from SFTP, extract CSV, and upload both to S3. Args: activity (ActivityWorker): The activity worker. feed_name (str): The name of the feed. date (str): Reporting date (YYYY-MM-DD). filename (str): Expected ZIP file name on the SFTP server. archive_path (str): S3 key for the raw ZIP archive. processed_path (str): S3 key for the extracted CSV file. """ activity.logger.info(f'Connecting to SFTP to download {filename}') sftp = _get_sftp_connection(config.sftp) remote_dir = config.sftp.get('path', '/FEED').rstrip('/') remote_file_path = f'{remote_dir}/{filename}' if remote_dir else filename try: sftp.stat(remote_file_path) except FileNotFoundError: activity.logger.info(f'File not available on SFTP: {filename}') garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE ) sftp.close() return {'stop': True, 'message': f'Missing file {filename}'} local_zip_path = os.path.join(tempfile.gettempdir(), filename) sftp.get(remote_file_path, local_zip_path) sftp.close() activity.logger.info(f'Downloaded {filename} to {local_zip_path}') s3_client = boto3.client('s3') activity.logger.info(f'Uploading ZIP to S3 archive: {archive_path}') s3_client.upload_file(local_zip_path, config.data_bucket, archive_path) activity.logger.info(f'Extracting CSV from ZIP: {filename}') with zipfile.ZipFile(local_zip_path) as zf: csv_names = [n for n in zf.namelist() if n.upper().endswith('.CSV')] if not csv_names: raise ValueError(f'No CSV file found inside ZIP: {filename}') csv_data = zf.read(csv_names[0]) os.remove(local_zip_path) activity.logger.info(f'Uploading extracted CSV to S3: {processed_path}') s3_client.put_object( Bucket=config.data_bucket, Key=processed_path, Body=csv_data, ) garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_DOWNLOADED ) activity.logger.info(f'Successfully processed {filename}')