"""Processes a Manifest object.""" import logging import os from backfill.connectors.ows_pdp.ows_pdp import OwsPdpClient from backfill.connectors.s3_connector import S3, FailedDownloadException from backfill.constants import JobType from backfill.job_processors import JobProcessor, MissingFileException from backfill.job_processors.attach_and_detach_processor import AttachAndDetachProcessor from backfill.models import Job, Manifest from backfill.utils import generate_local_filepath logger = logging.getLogger(__name__) class UnsupportedJobTypeException(Exception): pass SUPPORTED_JOB_PROCESSORS: dict[JobType, JobProcessor] = { JobType.JOB_TYPE_ATTACH_AND_DETACH: AttachAndDetachProcessor() } class ManifestProcessor: """Processes a Manifest object.""" def __init__( self, manifest: Manifest, ows_pdp_client: OwsPdpClient, s3_connector: S3, backfill_uuid: str, ) -> None: """Init method.""" self.manifest = manifest self.ows_pdp_client = ows_pdp_client self.s3_connector = s3_connector self.backfill_uuid = backfill_uuid def process(self) -> None: """Process jobs.""" for job in self.manifest.jobs: self._process_job(self.manifest.bucket, job) def _process_job(self, bucket: str, job: Job) -> None: """Process keys in job.""" for key in job.keys: self._process_file(bucket, key, job.job_type) def _process_file(self, bucket: str, key: str, job_type: JobType) -> None: """Process the file in job.""" job_processor = SUPPORTED_JOB_PROCESSORS.get(job_type, None) if not job_processor: raise UnsupportedJobTypeException(f"{job_type} is not supported") csv_filepath = generate_local_filepath(key) logger.info( "Downloading key to local filepath", extra={ "resources": { "bucket": bucket, "key": key, "csv_file": csv_filepath, "job_type": job_type.name, }, "correlation_id": self.backfill_uuid, }, ) try: # Use S3 Connector to download the file actual_filepath = self.s3_connector.download_file(bucket, key, csv_filepath) except FailedDownloadException as e: # Log the failure and return immediately logger.error( "Failed download from S3, continuing to next file in manifest", extra={ "resources": { "bucket": bucket, "key": key, "csv_file": csv_filepath, }, "correlation_id": self.backfill_uuid, }, exc_info=e, ) return try: job_processor.process( actual_filepath, self.ows_pdp_client, backfill_uuid=self.backfill_uuid, ) except MissingFileException as e: # Log the error and continue logger.error(e) # Clean-up - remove the file os.remove(actual_filepath)