import logging from dataclasses import dataclass from logging import Logger from typing import Optional, List from apollo_delphi_migration.exceptions import ValidationException from apollo_delphi_migration.renaming import RenamingStrategy from apollo_delphi_migration.s3 import RegexKeyMatcher, Bucket class MigrationTaskPostProcessor: def __init__(self, logger: logging.Logger) -> None: super().__init__() self._logger = logger def post_process(self, src_key: str, dst_key: str, context): raise NotImplementedError @dataclass(frozen=True) class MigrationContext: src_bucket: Bucket dst_bucket: Bucket renaming_strategies: List[RenamingStrategy] post_processors: List[MigrationTaskPostProcessor] class AbstractTask: def __init__(self, logger: Logger, src_key: str, dst_key: str, context: MigrationContext) -> None: super().__init__() self._logger = logger self.src_key = src_key self.dst_key = dst_key self._context = context def _execute(self) -> bool: raise NotImplementedError() def execute(self) -> bool: result = self._execute() for post_processor in self._context.post_processors: try: post_processor.post_process(self.src_key, self.dst_key, self._context) except ValidationException as ex: self._logger.warning( '[POST_PROCESSING][VALIDATION_FAILED] Validation failed during migration (%s -> %s): %s', self.src_key, self.dst_key, str(ex)) except Exception as ex: self._logger.error( '[POST_PROCESSING][FAILED] Exception raised during migration (%s -> %s) post processing: %s', self.src_key, self.dst_key, ex) return result class MigrationTask(AbstractTask): def _execute(self) -> bool: self._context.src_bucket.copy(self.src_key, self._context.dst_bucket.name, self.dst_key) return True class ValidationTask(AbstractTask): def _execute(self) -> bool: # Do nothing. Our goal is just to run post processors return True class AbstractTaskFactory: def __init__(self, logger: Logger, context: MigrationContext) -> None: super().__init__() self._logger = logger self._context = context def _create(self, key: str, renaming_strategy: RenamingStrategy) -> AbstractTask: raise NotImplementedError def create(self, key: str) -> AbstractTask: renaming_strategy = RenamingStrategy.select_renaming_strategy(self._context.renaming_strategies, key) if not renaming_strategy: self._logger.error('Unable to find renaming strategy for key. Key: `%a`', key) raise NotImplementedError('No suitable renaming strategy found.') return self._create(key, renaming_strategy) class MigrationTaskFactory(AbstractTaskFactory): def _create(self, key: str, renaming_strategy: RenamingStrategy) -> MigrationTask: return MigrationTask(self._logger, key, renaming_strategy.rename(key), self._context) class ValidationTaskFactory(AbstractTaskFactory): def _create(self, key: str, renaming_strategy: RenamingStrategy) -> AbstractTask: return ValidationTask(self._logger, key, renaming_strategy.rename(key), self._context) class ApolloReportLicensorKeyMatcher(RegexKeyMatcher): __REGEX_STR = '^.*/[a-z]+_[0-9-]{10}_#licensor_([0-9_]+)?v[0-9]+[\\._][0-9]+(_[a-z]+)?_#report\\.[a-z]+$' def __init__(self, report_type: Optional[str], licensor: Optional[str]) -> None: super().__init__(self.__REGEX_STR .replace('#licensor', licensor if licensor else '\\w+') .replace('#report', report_type if report_type else '\\w+')) class TheOrchardKeyMatcher(RegexKeyMatcher): __REGEX_STR = '^apple/(\\d{4}-\\d{2}-\\d{2})/AppleMusic_#report_([0-9]+)_\\d{4}\\d{2}\\d{2}\\.txt\\.gz$' def __init__(self, report_type: Optional[str]) -> None: report = report_type.lower().capitalize() if report_type else '\\w+' super().__init__(self.__REGEX_STR.replace('#report', report))