import csv import gzip from typing import Dict, Optional, Tuple from notification_service.db import NotificationDB, NotificationTable from notification_service.entities import DTFStatusEnum from notification_service.message_broker import SQSService import boto3 from datadog.dogstatsd.base import statsd from smart_open import smart_open from smelog.factory import SmeBoundLogger class SyncManager: stats_naming: str = NotImplemented def __init__(self, sqs_service: SQSService, db: NotificationDB, logger: SmeBoundLogger) -> None: self._sqs_service = sqs_service self._db = db self._logger = logger def run(self) -> Tuple: msgs = self._sqs_service.receive_msgs() success_msgs = 0 failed_msgs = 0 for receipt_handle, msg in msgs.items(): try: self._msg_handling(msg) except Exception as err: # pylint: disable=broad-except self._logger.error('[SYNC MANAGER RUN ERROR] - %s', err) failed_msgs += 1 else: self._sqs_service.remove_msg(receipt_handle=receipt_handle) success_msgs += 1 statsd.increment(f'notification-service.{self.stats_naming}_success_count', success_msgs) statsd.increment(f'notification-service.{self.stats_naming}_failed_count', failed_msgs) return success_msgs, failed_msgs def _msg_handling(self, msg: Dict) -> None: msg_row = self._db.get_notification(uow_id=msg['uow_id']) if msg_row: self._update_notification(row=msg_row, msg=msg) else: self._create_notification(msg=msg) def _update_notification(self, row: NotificationTable, msg: Dict) -> None: raise NotImplementedError() def _create_notification(self, msg: Dict) -> None: raise NotImplementedError() class DTFSyncService(SyncManager): stats_naming = 'sync_manager_dtf' def _update_notification(self, row: NotificationTable, msg: Dict) -> None: self._db.update_notification( notification_row=row, data_load_status=DTFStatusEnum(msg['status']).value, dtf_load_completed_at=msg['completed_at'], dtf_first_fact_id=msg['first_fact_id'], dtf_last_fact_id=msg['last_fact_id'], ) def _create_notification(self, msg: Dict) -> None: self._db.create_notification( uow_id=msg['uow_id'], dtf_first_fact_id=msg['first_fact_id'], dtf_last_fact_id=msg['last_fact_id'], data_load_status=DTFStatusEnum(msg['status']).value, dtf_load_completed_at=msg['completed_at'], ) class SFSyncService(SyncManager): stats_naming = 'sync_manager_sf' def _msg_handling(self, msg: Dict) -> None: msg = self._parse_msg(msg=msg) msg_row = self._db.get_notification(uow_id=msg['uow_id']) if msg_row: self._update_notification(row=msg_row, msg=msg) else: self._create_notification(msg=msg) def _update_notification(self, row: NotificationTable, msg: Dict) -> None: self._db.update_notification( notification_row=row, event_path=msg['event_path'], is_data_ready=msg['is_data_ready'], sf_load_completed_at=msg['sf_load_completed_at'], ) def _create_notification(self, msg: Dict) -> None: self._db.create_notification( uow_id=msg['uow_id'], event_path=msg['event_path'], is_data_ready=msg['is_data_ready'], sf_load_completed_at=msg['sf_load_completed_at'] ) def _parse_msg(self, msg: Dict) -> Dict: s3_info = msg['Records'][0]['s3'] bucket_name = s3_info['bucket']['name'] success_file_path = s3_info['object']['key'].replace('%3D', '=') general_file_path = '/'.join(success_file_path.split('/')[:-1]) uow_id = int( [fp.split('=') for fp in success_file_path.split('/') if 'unit_of_work=' in fp][0][1] ) is_data_ready, sf_load_completed_at = self._check_completeness_data( bucket_name=bucket_name, success_file_path=success_file_path, general_file_path=general_file_path, ) return { 'event_path': f's3://{bucket_name}/{general_file_path}/', 'uow_id': uow_id, 'is_data_ready': is_data_ready, 'sf_load_completed_at': sf_load_completed_at } @staticmethod def _check_completeness_data(bucket_name: str, success_file_path: str, general_file_path: str) -> Tuple[bool, Optional[str]]: notification_bucket = boto3.resource('s3').Bucket(bucket_name) files_in_s3 = [] for object_summary in notification_bucket.objects.filter(Prefix=general_file_path): files_in_s3.append(object_summary.key.split('/')[-1]) with smart_open(f's3://{bucket_name}/{success_file_path}') as archive: with gzip.open(archive, 'rt') as fo: # pylint: disable=invalid-name data_frame = csv.reader(fo, delimiter='^') next(data_frame, None) # skip header export_completed_utc = None for row in data_frame: file_name = row[0] export_completed_utc = row[1] if file_name not in files_in_s3: return False, export_completed_utc return True, export_completed_utc