from loguru import logger from ..config import QUERY_PATHS, TABLES, WAREHOUSES from ..db import get_snw, get_rdb, get_s3 from ..db.queries import QueryLoader def run_sync() -> None: logger.info("Starting Snowflake-to-Redshift sync") snw = get_snw(WAREHOUSES["large"]) ql = QueryLoader() # Step 1: Create the diff table (changed ISRCs between main and mirror) logger.info("Creating changed table (diff between main and redshift mirror)") snw.execute(ql.load(QUERY_PATHS.changed_create)) # Step 2: Query changed data BEFORE touching the mirror logger.info("Querying changed ISRCs from Snowflake") redshift_changed = TABLES["redshift_changed"] snowflake_changed = snw.query(f"select * from {redshift_changed}") if len(snowflake_changed) == 0: logger.info("No new or changed ISRCs — nothing to sync") return logger.info(f"Found {len(snowflake_changed)} new/changed ISRCs") # Step 3: Write to Redshift rdb = get_rdb() s3 = get_s3() rdb_main = TABLES["rdb_main"] rdb_changed = TABLES["rdb_changed"] logger.info("Truncating Redshift staging table") rdb.execute(f"truncate table {rdb_changed}") logger.info("Uploading changed data to Redshift via S3") s3.write_df_to_reportingDB(snowflake_changed, rdb_changed) logger.info("Applying changes to Redshift main table") rdb.execute( f"delete from {rdb_main} " f"where isrc_code in (select isrc_code from {rdb_changed})" ) rdb.execute(f"insert into {rdb_main} select * from {rdb_changed}") # Step 4: Only AFTER successful Redshift write, rebuild the mirror as a # full snapshot of main. This ensures: # - If Redshift write fails, the mirror is unchanged and the diff will # be re-detected on the next run (no data loss). # - The mirror always matches main exactly, preventing row count drift # from full refreshes that remove ISRCs from main. logger.info("Rebuilding Snowflake mirror table as full snapshot of main") redshift_current = TABLES["redshift_current"] main_table = TABLES["main"] snw.execute( f"create or replace table {redshift_current} as " f"select isrc_code, isrc_origin from {main_table}" ) logger.success(f"Synced {len(snowflake_changed)} ISRCs to Redshift")