""" lambda-d2c-connect-shopify-stores Triggered hourly by EventBridge. Queries ARTIST_STORES for myshopify domains not yet in onboarding state or the store registry, filters out CRM and Fansifter managed stores, and pushes each new domain to SQS for onboarding. """ import json import logging import boto3 from datadog_lambda.metric import lambda_metric import config from src.snowflake_client import ( get_connection, get_artist_stores_metadata, get_onboarding_domains, get_registry_domains, get_crm_domains, get_fansifter_domains, upsert_registry, ) from src.store_utils import normalise_url, derive_schema_name logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) _sqs = boto3.client("sqs", region_name=config.AWS_REGION) _REGISTRY_SOURCE = "D2C" def _find_new_stores(sf) -> list[dict]: already_tracked = get_onboarding_domains(sf) | get_registry_domains(sf) crm_domains = get_crm_domains(sf) fansifter_domains = get_fansifter_domains(sf) excluded = already_tracked | crm_domains | fansifter_domains rows = get_artist_stores_metadata(sf) new_stores = [] seen_domains: set[str] = set() for row in rows: seen_in_row: set[str] = set() domains: list[str] = [] for f in ("shopify_shop", "shopify_store_url", "public_domain", "alternate_domain"): val = row.get(f) if val and val.strip(): norm = normalise_url(val) if norm.endswith(".myshopify.com") and norm not in seen_in_row: seen_in_row.add(norm) domains.append(norm) if not domains: continue # If any domain for this row is already tracked, skip the whole row — # shopify_shop and shopify_store_url can differ for the same store. if any(d in excluded or d in seen_domains for d in domains): implicit = [d for d in domains if d not in excluded and d not in seen_domains] if implicit: trigger = next((d for d in domains if d in excluded), None) or next( d for d in domains if d in seen_domains ) reason = "already excluded" if trigger in excluded else "already seen in this run" logger.warning( f"Suppressing {implicit} — co-listed on the same ARTIST_STORES row with {trigger!r} ({reason})" ) seen_domains.update(domains) continue canonical = domains[0] alt_myshopify_domain = domains[1] if len(domains) > 1 else None seen_domains.update(domains) custom_domain = None for f in ("public_domain", "alternate_domain"): raw = (row.get(f) or "").strip() if raw: norm = normalise_url(raw) if not norm.endswith(".myshopify.com"): custom_domain = norm break country = (row.get("selling_country") or "").strip() if country.startswith("#"): country = "" schema_name = derive_schema_name(canonical, country, row.get("rep_owner"), row.get("merch_company")) new_stores.append( { "shop_domain": canonical, "schema_name": schema_name, "merch_company": (row.get("merch_company") or "").strip() or None, "selling_country": country or None, "rep_owner": (row.get("rep_owner") or "").strip() or None, "alt_myshopify_domain": alt_myshopify_domain, "custom_domain": custom_domain, "is_active": row.get("active_"), } ) return new_stores def handler(event, context): sf = get_connection() try: new_stores = _find_new_stores(sf) if not new_stores: lambda_metric("d2c.connect_shopify_stores.discovered", 0) return {"discovered": 0} logger.info(f"Discovered {len(new_stores)} new store(s)") lambda_metric("d2c.connect_shopify_stores.discovered", len(new_stores)) queued = 0 send_failed = 0 registry_write_failed = 0 for store in new_stores: domain = store["shop_domain"] # Enqueue before writing the registry row. If the SQS send fails we # have written nothing, so the store is simply rediscovered next run # (no orphan). If the send succeeds but the registry upsert fails, the # store is still queued and Lambda 2 will onboard it — its registry # row just stays absent until it is written at sync_complete. It is # NOT rediscovered/re-queued: once Lambda 2 writes onboarding state, # get_onboarding_domains excludes it from discovery. The two writes # are handled separately so the store counts as queued the moment the # send succeeds, and a registry-write failure is reported distinctly. try: _sqs.send_message( QueueUrl=config.SQS_QUEUE_URL, MessageBody=json.dumps(store), ) except Exception as exc: send_failed += 1 logger.error(f"[{domain}] SQS send failed — not queued: {exc}", exc_info=True) continue queued += 1 logger.debug("Queued: %s -> %s", store["shop_domain"], store["schema_name"]) try: upsert_registry( sf, { "myshopify_domain": store["shop_domain"], "schema_name": store["schema_name"], "source": _REGISTRY_SOURCE, "merch_company": store.get("merch_company"), "selling_country": store.get("selling_country"), "rep_owner": store.get("rep_owner"), "alt_myshopify_domain": store.get("alt_myshopify_domain"), "custom_domain": store.get("custom_domain"), "is_active": store.get("is_active"), "is_synced_to_snowflake": False, }, ) except Exception as exc: registry_write_failed += 1 logger.error( f"[{domain}] Queued, but registry upsert failed (row will be written at sync_complete): {exc}", exc_info=True, ) finally: sf.close() if send_failed: lambda_metric("d2c.connect_shopify_stores.send_failed", send_failed) if registry_write_failed: lambda_metric("d2c.connect_shopify_stores.registry_write_failed", registry_write_failed) logger.info(f"Queued {queued} store(s); send_failed={send_failed}, registry_write_failed={registry_write_failed}") return { "discovered": len(new_stores), "queued": queued, "send_failed": send_failed, "registry_write_failed": registry_write_failed, }