import json from typing import Any, List import structlog from db_schema.schemas.apps import ActivityStatusEnum as AppsActivityStatusEnum from db_schema.schemas.apps import CompletenessStatusEnum as AppsCompletenessStatusEnum from db_schema.schemas.apps import DatabricksExecution, DBXJobStatusEnum from db_schema.schemas.apps import UnitOfWork as AppsUnitOfWork from db_schema.schemas.apps import UnitOfWorkPriorityEnum as AppsUnitOfWorkPriorityEnum from db_schema.schemas.apps import UnitOfWorkTypeEnum as AppsUnitOfWorkTypeEnum from flask import current_app, flash, request from flask_admin.actions import action from flask_admin.contrib.sqla.filters import BaseSQLAFilter from sqlalchemy import and_, inspect, literal, or_ from sqlalchemy.orm import Query, Session from delphi_slz_admin.const import APP_NAME from delphi_slz_admin.helpers import cast_ids_to_int, utcnow from delphi_slz_admin.modelviews import ( BaseFilterByDateEarlierThan, BaseFilterByDateEquals, BaseFilterByDateLaterThan, BaseFilterById, BaseFilterByIdReversed, BaseModelView, ) from delphi_slz_admin.services.audit_log import AuditLogService from delphi_slz_admin.services.config import Config from delphi_slz_admin.services.reprocessing import ReprocessingService logger = structlog.get_logger(APP_NAME) class FilterUnitOfWorkId(BaseFilterById): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.unit_of_work_id self.name = 'Unit of Work ID' super().__init__(self.column, self.name, **kwargs) class FilterUnitOfWorkIdReversed(BaseFilterByIdReversed): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.unit_of_work_id self.name = 'Unit of Work ID' super().__init__(self.column, self.name, **kwargs) class FilterUnitOfWorkType(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.unit_of_work_type self.name = 'Unit of Work Type' self.options = [(item.value, item.value) for item in AppsUnitOfWorkTypeEnum] super().__init__(self.column, self.name, options=self.options, **kwargs) def apply(self, query: Query, value: str, alias: Any = None) -> Query: return query.filter(self.column == value) def operation(self) -> str: return 'equals' class FilterCompletenessStatus(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.completeness_status self.name = 'Completeness Status' self.options = [(item.value, item.value) for item in AppsCompletenessStatusEnum] super().__init__(self.column, self.name, options=self.options, **kwargs) def apply(self, query: Query, value: str, alias: Any = None) -> Query: return query.filter(self.column == value) def operation(self) -> str: return 'equals' class FilterActivityStatus(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.activity_status self.name = 'Activity Status' self.options = [(item.value, item.value) for item in AppsActivityStatusEnum] super().__init__(self.column, self.name, options=self.options, **kwargs) def apply(self, query: Query, value: str, alias: Any = None) -> Query: return query.filter(self.column == value) def operation(self) -> str: return 'equals' class FilterPriority(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.priority self.name = 'Priority' super().__init__(self.column, self.name, **kwargs) def clean(self, value: str) -> int: return int(value) def apply(self, query: Query, value: str, alias: Any = None) -> Query: return query.filter(self.column <= value) def operation(self) -> str: return 'less or equal' class FilterLatestJobIdIsNone(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.latest_job_id self.name = 'Latest Job ID' self.options = [(None, None)] super().__init__(self.column, self.name, options=self.options, **kwargs) def apply(self, query: Query, value: Any, alias: Any = None) -> Query: return query.filter(self.column.is_(None)) def operation(self) -> str: return 'is' class FilterLatestJobIdIsNotNone(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.latest_job_id self.name = 'Latest Job ID' self.options = [(None, None)] super().__init__(self.column, self.name, options=self.options, **kwargs) def apply(self, query: Query, value: Any, alias: Any = None) -> Query: return query.filter(self.column.isnot(None)) def operation(self) -> str: return 'is not' class FilterReportDateMixin(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.report_date self.name = 'Report date' super().__init__(self.column, self.name, data_type='datepicker', **kwargs) def operation(self) -> str: pass class FilterReportDateLaterThan(BaseFilterByDateLaterThan, FilterReportDateMixin): pass class FilterReportDateEarlierThan(BaseFilterByDateEarlierThan, FilterReportDateMixin): pass class FilterReportDateEquals(BaseFilterByDateEquals, FilterReportDateMixin): pass class FilterDbxExecutionIdIsNone(BaseSQLAFilter): def __init__(self, **kwargs: Any) -> None: self.column = AppsUnitOfWork.latest_job_id self.name = 'DBX Execution ID' self.options = [(None, None)] super().__init__(self.column, self.name, options=self.options, **kwargs) def apply(self, query: Query, value: Any, alias: Any = None) -> Query: return query.outerjoin( DatabricksExecution, and_( DatabricksExecution.unit_of_work_id == AppsUnitOfWork.unit_of_work_id, DatabricksExecution.status == DBXJobStatusEnum.COMPLETE, ) ).filter(DatabricksExecution.databricks_execution_id.is_(None)) def operation(self) -> str: return 'is' class AppsUnitOfWorkModelView(BaseModelView): truncate_limit = 80 column_list = [c_attr.key for c_attr in inspect(AppsUnitOfWork).mapper.column_attrs] column_sortable_list = column_list column_filters = [ FilterUnitOfWorkId(), FilterUnitOfWorkIdReversed(), FilterUnitOfWorkType(), FilterCompletenessStatus(), FilterActivityStatus(), FilterPriority(), FilterLatestJobIdIsNone(), FilterLatestJobIdIsNotNone(), FilterReportDateEarlierThan(), FilterReportDateLaterThan(), FilterReportDateEquals(), FilterDbxExecutionIdIsNone(), ] list_template = 'apps_unit_of_work_list.html' def __init__(self, session: Session, **kwargs: Any) -> None: self.model = AppsUnitOfWork self.name = 'ETL Unit of Work' self.endpoint = 'etl_unit_of_work' self.audit_log_service = AuditLogService(session) self.reprocessing_service = ReprocessingService(session) super().__init__(self.model, session, name=self.name, endpoint=self.endpoint, **kwargs) def _get_latest_job_id(self, instance: AppsUnitOfWork) -> str: """This part of code is similar to https://github.com/filtr/delphi-etl-orchestration/blob/develop/apps-etl/lambda/apps_etl_mana ger/apps_etl_manager/unit_of_works/base.py#L55""" current_dt_str = utcnow().strftime('%Y%m%dT%H.%M.%S') sf_execution_name = f'_{instance.unit_of_work_id}_{current_dt_str}' chars_left = self.truncate_limit - len(sf_execution_name) return f'{instance.unit_of_work_code[:chars_left]}{sf_execution_name}' @action( 'restart_failed_backfill', 'Restart failed backfill', 'Are you sure you want to restart failed backfill?' ) def action_restart_failed_backfill(self, ids: List[str]) -> None: try: casted_ids = cast_ids_to_int(ids, logger) query = self.session.query(self.model).filter( self.model.unit_of_work_id.in_(casted_ids) ) has_improper_status = self.session.query(literal(True)).filter( query.filter( or_( AppsUnitOfWork.completeness_status != AppsCompletenessStatusEnum.FAILED, AppsUnitOfWork.unit_of_work_type != AppsUnitOfWorkTypeEnum.BACKFILL, ) ).exists() ).scalar() if has_improper_status: flash( f'Some items are not supposed to be restarted. Completeness Status should be ' f'equal to {AppsCompletenessStatusEnum.FAILED.value}. ' f'UoW Type should be equal to {AppsUnitOfWorkTypeEnum.BACKFILL.value}.', category='error' ) return apps_uow_list: List[AppsUnitOfWork] = query.all() for apps_uow in apps_uow_list: apps_uow.completeness_status = AppsCompletenessStatusEnum.QUEUED apps_uow.last_updated_at = utcnow() apps_uow.next_run_at = utcnow() apps_uow.latest_job_id = self._get_latest_job_id(apps_uow) self.audit_log_service.log_apps_unit_of_work_restart_failed_backfill(casted_ids) self.session.commit() flash(f'{len(apps_uow_list)} ETL UoW(es) were successfully restarted.') except Exception as ex: # pylint: disable=broad-except if not self.handle_view_exception(ex): raise logger.error('Failed to restart ETL UoW, details = %s', ex) flash('Failed to restart ETL UoW', category='error') @action('reprocess', 'Reprocess Unit of Work') def action_reprocess(self, ids: List[str]) -> None: # pylint: disable=too-many-locals raw_priority = request.values.get('priority') raw_driver_options = request.values.get('driver_options') try: if raw_priority == '': priority = None else: try: # Consider that the highest priority has the lowest integer value priority = AppsUnitOfWorkPriorityEnum(int(raw_priority)) lowest = AppsUnitOfWorkPriorityEnum.LOWEST.value highest = AppsUnitOfWorkPriorityEnum.HIGHEST.value if not highest <= priority.value <= lowest: raise ValueError except (ValueError, TypeError): flash('Invalid value for Priority', category='error') return try: driver_options = json.loads(raw_driver_options) if raw_driver_options else None except (ValueError, TypeError): flash('Invalid value for Driver Options', category='error') return casted_ids = cast_ids_to_int(ids, logger) query = self.session.query(self.model).filter( self.model.unit_of_work_id.in_(casted_ids) ) rejected_statuses = [ AppsCompletenessStatusEnum.ACTIVE, AppsCompletenessStatusEnum.MIN_COMPLETE, AppsCompletenessStatusEnum.QUEUED ] has_improper_status = self.session.query(literal(True)).filter( query.filter(AppsUnitOfWork.completeness_status.in_(rejected_statuses)).exists() ).scalar() if has_improper_status: joined_rejected_statuses = ', '.join(map(lambda x: x.value, rejected_statuses)) flash( f'Some items are not supposed to be reprocessed. Completeness Status should not' f' be one of {joined_rejected_statuses}.', category='error' ) return config: Config = current_app.config['CONFIG'] count = self.reprocessing_service.reprocess_apps_uows( casted_ids, priority, driver_options, config.env ) self.audit_log_service.log_apps_unit_of_work_reprocessed(casted_ids) self.session.commit() flash(f'{count} ETL Unit(s) of Work were successfully marked for reprocessing') except Exception as ex: # pylint: disable=broad-except if not self.handle_view_exception(ex): raise logger.error('Failed to perform reprocessing, details = %s', ex) flash('Failed to perform reprocessing, see logs for details', category='error')