import csv import json import os from typing import TYPE_CHECKING import boto3 if TYPE_CHECKING: from mypy_boto3_sqs.type_defs import SendMessageBatchRequestEntryTypeDef ENVIRONMENT = os.environ.get("ENVIRONMENT", "qa") CSV_FILE = os.environ["CSV_FILE"] START_OFFSET = int(os.environ.get("START_OFFSET", 0)) # Set OVERWRITE=true to re-scan assets that already have a scan: each message body # carries an "OVERWRITE" flag that the hive_ai_detection lambda (lambda-assets) reads # to post with overwrite, so ows-assets replaces the existing rows instead of 409ing. # The "OVERWRITE" key name is a contract with that consumer. Fail loud on any value # other than true/false so a typo (e.g. OVERWRITE=1) can't silently run a no-op. _overwrite = os.environ.get("OVERWRITE", "false").lower() if _overwrite not in {"true", "false"}: raise SystemExit(f"OVERWRITE must be 'true' or 'false', got {_overwrite!r}") OVERWRITE = _overwrite == "true" QUEUE_NAME = f"{ENVIRONMENT}-hive-ai-detection-backfill-queue" SEND_MESSAGE_BATCH_MAXIMUM = 10 PROGRESS_INTERVAL_MESSAGES = 10_000 def main() -> None: sqs = boto3.client("sqs") queue_url = sqs.get_queue_url(QueueName=QUEUE_NAME)["QueueUrl"] with open(f"input/{CSV_FILE}") as csv_file: asset_rows = list(csv.DictReader(csv_file)) total = len(asset_rows) print( f"{total} assets in {CSV_FILE}; starting at offset {START_OFFSET}; " f"overwrite={OVERWRITE}" ) overwrite_field = {"OVERWRITE": True} if OVERWRITE else {} enqueued_message_total = 0 for offset in range(START_OFFSET, total, SEND_MESSAGE_BATCH_MAXIMUM): batch = asset_rows[offset : offset + SEND_MESSAGE_BATCH_MAXIMUM] entries: list[SendMessageBatchRequestEntryTypeDef] = [ { "Id": str(index), "MessageBody": json.dumps( { "ASSET_FINAL_ID": int(row["ID"]), "DURATION_MS": int(row["DURATION"]), **overwrite_field, } ), } for index, row in enumerate(batch) ] response = sqs.send_message_batch(QueueUrl=queue_url, Entries=entries) if response.get("Failed"): # rerun with START_OFFSET= to resume; duplicates from the # partial batch re-scan and then 409 at the save step — bounded to # one batch, so the wasted cost is negligible raise RuntimeError( f"send_message_batch failed at offset {offset}: {response['Failed']}" ) enqueued_message_total += len(batch) enqueued = offset + len(batch) if enqueued % PROGRESS_INTERVAL_MESSAGES < SEND_MESSAGE_BATCH_MAXIMUM: print(f"{enqueued}/{total} enqueued") print(f"Done: {enqueued_message_total} messages enqueued to {QUEUE_NAME}")