from loguru import logger from ..config import QUERY_PATHS, TABLES, WAREHOUSES from ..db import get_snw, get_rdb from ..db.queries import QueryLoader def run_sync() -> None: """Push new exchange rates from Snowflake to Redshift.""" logger.info("Starting Snowflake-to-Redshift sync") rdb = get_rdb() snw = get_snw(WAREHOUSES["small"]) ql = QueryLoader() # Step 0: Ensure Redshift table exists rdb.execute(ql.load(QUERY_PATHS.rdb_ensure)) logger.debug("Ensured Redshift table exists") # Step 1: Get high-water mark from Redshift logger.info("Querying Redshift for max rate_date") cutoff_df = rdb.query(ql.load(QUERY_PATHS.rdb_get_cutoff)) cutoff = cutoff_df["max_date"][0] if cutoff is None: logger.info("Redshift table is empty — syncing all rows from Snowflake") else: logger.info(f"Redshift cutoff: {cutoff}") # Step 2: Get new rows from Snowflake newdata = snw.query(ql.load(QUERY_PATHS.snw_get_new, cutoff=cutoff)) if len(newdata) == 0: logger.info("No new rates to sync") return # Step 3: Write to Redshift logger.info(f"Appending {len(newdata)} rate rows to Redshift") rdb.write(newdata, TABLES["rdb"]) logger.success(f"Synced {len(newdata)} rate rows to Redshift")