import boto3 import csv import sys S3_BUCKET = "prod-vector-audit" S3_PREFIX = "metadata" def fetch_xml_from_s3(s3_client, bucket, key): try: response = s3_client.get_object(Bucket=bucket, Key=key) return response["Body"].read().decode("utf-8") except s3_client.exceptions.NoSuchKey: print(f" [WARN] File not found in S3: s3://{bucket}/{key}") return None except Exception as e: print(f" [ERROR] Failed to fetch s3://{bucket}/{key}: {e}") return None def main(csv_path): s3_client = boto3.client("s3") missing_upc_rows = [] with open(csv_path, newline="", encoding="utf-8") as csvfile: reader = csv.DictReader(csvfile) required_headers = {"ENCODING_QUEUE_DETAIL_ID", "FILENAME", "UPC"} if not required_headers.issubset(reader.fieldnames): print(f"[ERROR] CSV is missing required headers. Expected: {required_headers}") sys.exit(1) for row in reader: detail_id = row["ENCODING_QUEUE_DETAIL_ID"].strip() filename = row["FILENAME"].strip() upc = row["UPC"].strip() s3_key = f"{S3_PREFIX}/{filename}" patterns = [f"{upc}_", f"{upc}.xml", f'{upc}', f'{upc}'] print(f"Checking ENCODING_QUEUE_DETAIL_ID={detail_id}, FILE={filename}, UPC={upc}") xml_content = fetch_xml_from_s3(s3_client, S3_BUCKET, s3_key) if xml_content is None: print(f" [SKIP] Could not retrieve XML, skipping row.") missing_upc_rows.append({**row, "REASON": "S3 file not found or error"}) continue found = any(pattern in xml_content for pattern in patterns) if not found: print(f" [MISSING] Neither '{upc}_' nor '{upc}.xml' found in {filename}") missing_upc_rows.append({**row, "REASON": "All patterns not found in XML"}) else: matched = [p for p in patterns if p in xml_content] print(f" [OK] Found pattern(s) {matched} in {filename}") print(f"\n{'='*60}") print(f"Done. {len(missing_upc_rows)} row(s) with missing UPCs:\n") if missing_upc_rows: output_path = "missing_upc_results.csv" fieldnames = ["ENCODING_QUEUE_DETAIL_ID", "FILENAME", "UPC", "REASON"] with open(output_path, "w", newline="", encoding="utf-8") as out_csv: writer = csv.DictWriter(out_csv, fieldnames=fieldnames) writer.writeheader() writer.writerows(missing_upc_rows) for row in missing_upc_rows: print(f" - ID={row['ENCODING_QUEUE_DETAIL_ID']}, FILE={row['FILENAME']}, UPC={row['UPC']} ({row['REASON']})") print(f"\nResults written to: {output_path}") else: print(" All UPCs were found!") if __name__ == "__main__": if len(sys.argv) != 2: print("Usage: python check_metadata.py ") sys.exit(1) main(sys.argv[1])