import boto3
import csv
import sys
S3_BUCKET = "prod-vector-audit"
S3_PREFIX = "metadata"
def fetch_xml_from_s3(s3_client, bucket, key):
try:
response = s3_client.get_object(Bucket=bucket, Key=key)
return response["Body"].read().decode("utf-8")
except s3_client.exceptions.NoSuchKey:
print(f" [WARN] File not found in S3: s3://{bucket}/{key}")
return None
except Exception as e:
print(f" [ERROR] Failed to fetch s3://{bucket}/{key}: {e}")
return None
def main(csv_path):
s3_client = boto3.client("s3")
missing_upc_rows = []
with open(csv_path, newline="", encoding="utf-8") as csvfile:
reader = csv.DictReader(csvfile)
required_headers = {"ENCODING_QUEUE_DETAIL_ID", "FILENAME", "UPC"}
if not required_headers.issubset(reader.fieldnames):
print(f"[ERROR] CSV is missing required headers. Expected: {required_headers}")
sys.exit(1)
for row in reader:
detail_id = row["ENCODING_QUEUE_DETAIL_ID"].strip()
filename = row["FILENAME"].strip()
upc = row["UPC"].strip()
s3_key = f"{S3_PREFIX}/{filename}"
patterns = [f"{upc}_", f"{upc}.xml", f'{upc}', f'{upc}']
print(f"Checking ENCODING_QUEUE_DETAIL_ID={detail_id}, FILE={filename}, UPC={upc}")
xml_content = fetch_xml_from_s3(s3_client, S3_BUCKET, s3_key)
if xml_content is None:
print(f" [SKIP] Could not retrieve XML, skipping row.")
missing_upc_rows.append({**row, "REASON": "S3 file not found or error"})
continue
found = any(pattern in xml_content for pattern in patterns)
if not found:
print(f" [MISSING] Neither '{upc}_' nor '{upc}.xml' found in {filename}")
missing_upc_rows.append({**row, "REASON": "All patterns not found in XML"})
else:
matched = [p for p in patterns if p in xml_content]
print(f" [OK] Found pattern(s) {matched} in {filename}")
print(f"\n{'='*60}")
print(f"Done. {len(missing_upc_rows)} row(s) with missing UPCs:\n")
if missing_upc_rows:
output_path = "missing_upc_results.csv"
fieldnames = ["ENCODING_QUEUE_DETAIL_ID", "FILENAME", "UPC", "REASON"]
with open(output_path, "w", newline="", encoding="utf-8") as out_csv:
writer = csv.DictWriter(out_csv, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(missing_upc_rows)
for row in missing_upc_rows:
print(f" - ID={row['ENCODING_QUEUE_DETAIL_ID']}, FILE={row['FILENAME']}, UPC={row['UPC']} ({row['REASON']})")
print(f"\nResults written to: {output_path}")
else:
print(" All UPCs were found!")
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python check_metadata.py ")
sys.exit(1)
main(sys.argv[1])