from airflow.models import DAG from airflow.operators.bash import BashOperator from airflow.utils.dates import days_ago, timedelta from fansifter.callbacks.slack import on_dag_failure_slack_callback AIRFLOW_EFS_DIRECTORY = "/mnt/efs/fansifter-airflow" default_dag_args = dict(start_date=days_ago(1), owner="fansifter") with DAG( dag_id="git_pull_dag", schedule_interval=timedelta(minutes=60), default_args=default_dag_args, catchup=False, on_failure_callback=on_dag_failure_slack_callback, ) as dag: bash_command = f""" if [ $IS_LOCAL = "true" ]; then echo "Running Airflow locally, no need to pull DAGs and plugins from remote..." exit 0 fi case $STAGE in dev) BRANCH="master" ;; test) BRANCH="deploy_test" ;; live) BRANCH="deploy_live" ;; *) exit 1 ;; esac export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" cd {AIRFLOW_EFS_DIRECTORY} git fetch origin $BRANCH git checkout origin/$BRANCH -- dags git checkout origin/$BRANCH -- plugins """ git_pull = BashOperator(task_id="git-pull", bash_command=bash_command)