import json from datetime import datetime, timedelta from dbt.cli.main import dbtRunner, dbtRunnerResult def generate_dates(from_date: str, to_date: str, interval: int) -> [str]: """ Function to create list of dates using specified interval :param from_date: Starting date inclusive :param to_date: Last date exclusive :param interval: How many days between each date :return: List of dates in string format """ start_date = datetime.strptime(from_date, "%Y-%m-%d") end_date = datetime.strptime(to_date, "%Y-%m-%d") dates = [] current_date = start_date while current_date <= end_date: dates.append(current_date.strftime("%Y-%m-%d")) current_date += timedelta(days=interval) return dates def run_dbt_command(backfill_date: str, gp_id: str, vendor: int, subaccount: int) -> None: """ Function t0 trigger regular DBT commands :param backfill_date: Starting date inclusive :param gp_id: Global participant id :param vendor: Vendor id :param subaccount: Subaccount id :return: None """ print(f"Starting dbt run using date: {backfill_date}:") # initialize dbt = dbtRunner() date_variable_short = backfill_date date_variable = backfill_date + " 00:00:00" vars_dict = {"date_limit": date_variable, "short_dt": date_variable_short, "gp_id": gp_id, "vendor": vendor, "subaccount": subaccount, } vars_json = json.dumps(vars_dict) print(f"Using following vars {vars_json}") # create CLI args as a list of strings cli_args = [ "run", "--select", "models/backfill", "--vars", vars_json, "--profile", "dbt_sme", ] # run the command res: dbtRunnerResult = dbt.invoke(cli_args) # inspect the results for r in res.result: print(f"{r.node.name}: {r.status}") def run_dbt_snapshot() -> None: """ Function to trigger DBT snapshot command :return: None """ # initialize dbt = dbtRunner() cli_args = [ "snapshot", "--select", "BF_FAN_SEGMENT_SNAPSHOT", "--profile", "dbt_sme", ] # run the command res: dbtRunnerResult = dbt.invoke(cli_args) # inspect the results for r in res.result: print(f"{r.node.name}: {r.status}") if __name__ == "__main__": # Fill in below variables: start_date = "2024-04-07" end_date = "2024-04-22" gp_id = 'a5de08a1-d124-47aa-be84-f96dfb771833' vendor=34603 subaccount=0 days_between_backfills = 30 date_list = generate_dates(start_date, end_date, days_between_backfills) print(f"running backfilling using following dates: {date_list}") for d in date_list: run_dbt_command(d, gp_id, vendor, subaccount) # run_dbt_snapshot()