"""Ingest Logic.""" import re from typing import Any import boto3 from sqlalchemy import func from video import config from video.constants.ingest import DEFAULT_BATCH_SIZE, INGEST_INCOMING_PATH from video.logic import job as job_logic from video.models.sql.classes import product_video def ingest(data: dict[str, Any]) -> list[dict[str, Any]]: """Check if there are files to ingest and if so setup the workflows. Args: data (dict): Parameters to configure the ingest. data['batch_size'] (int): The number of products to ingest. Returns: list: the files that will be ingested. """ batch_size = data.get("batch_size", DEFAULT_BATCH_SIZE) files = _filter_files_with_product(_get_files_to_ingest()) files = files[:batch_size] for file in files: jobs = job_logic.setup_ingest_from_s3_workflow(file) file["latest_pipeline_run_id"] = jobs[0]["parent_id"] product_video.upsert({**file, "migrated_asset_at": func.now()}) return files def _get_files_to_ingest() -> list[dict[str, Any]]: """Get the list of files to ingest from S3. Returns: [dict]: containing the list of files to ingest. """ result = boto3.client("s3").list_objects_v2( Bucket=config.VIDEO_BUCKET_NAME, Prefix=INGEST_INCOMING_PATH ) files = [] for item in result.get("Contents", []): key = item["Key"] found_filename = re.search(r"[^/]+\.mov$", key) if found_filename: files.append({"path": key, "filename": found_filename.group(0)}) return files def _filter_files_with_product(files: list[dict[str, Any]]) -> list[dict[str, Any]]: """Filter the files with an existing product_video from a list of files. Args: files ([dict]): The list of files. Returns: [dict]: containing the filtered list. """ files_with_product = [] for file in files: product_response = product_video.get_by_ingest_filename(file["filename"]) # Filter files that don't have a row in product_video if product_response: # Filter products that have already been through the pipeline if not product_response.get("latest_pipeline_run_id"): file["release_id"] = product_response["release_id"] files_with_product.append(file) return files_with_product