import boto3 import json from pprint import pprint import requests from requests.auth import HTTPBasicAuth from datetime import datetime, timezone import config from dotenv import load_dotenv import os import uuid import snow load_dotenv() aws_dev_session = boto3.Session( aws_access_key_id=os.environ.get("DEV_AWS_ACCESS_KEY_ID"), aws_secret_access_key=os.environ.get("DEV_AWS_SECRET_ACCESS_KEY"), aws_session_token=os.environ.get("DEV_AWS_SESSION_TOKEN"), ) s3_client = aws_dev_session.client("s3") def get_jenkins_builds(job): fields = "builds[number,result,url,queueId,building,timestamp,duration,actions[parameters[name,value]]]" limit = "{0,30}" url = f"{config.jenkins_base_url}/job/{job}/api/json?tree={fields}{limit}" response = requests.get( url, auth=HTTPBasicAuth(config.jenkins_username, config.jenkins_secret), timeout=5, ) if response.status_code != 200: response.raise_for_status() builds = [] body = response.json() for b in body["builds"]: build_status = b["result"] if b["building"] is True: build_status = "IN_PROGRESS" build = { "job": job, "build_id": b["number"], "status": build_status, "parameters": None, } for a in b["actions"]: if a.get("parameters") is not None: build["parameters"] = {} for p in a["parameters"]: build["parameters"][p["name"]] = p["value"] start_time_sec = b["timestamp"] / 1000 start_time_dt = datetime.fromtimestamp(start_time_sec, tz=timezone.utc) build["start_time"] = start_time_dt.isoformat() if b["building"] is False: end_time_sec = (b["timestamp"] + b["duration"]) / 1000 end_time_dt = datetime.fromtimestamp(end_time_sec, tz=timezone.utc) build["end_time"] = end_time_dt.isoformat() builds.append(build) return builds def jenkins(): filename = f"{str(uuid.uuid4())}.json" dbt_builds = get_jenkins_builds("dbt-scheduler-analytics-pipeline") chart_builds = get_jenkins_builds("swf-chartmetric-charts-exec") builds = dbt_builds + chart_builds body = json.dumps(builds) s3_client.put_object( Body=body, Bucket="dev-cucumbers", Key=f"FeedIngestionStatusAudit/jenkins/{filename}", ContentType="application/json", ) truncate_staging_query = ( "TRUNCATE TABLE dev_engineering.sduberg.temp_staging_raw_audit_jenkins" ) copy_into_query = f""" COPY INTO dev_engineering.sduberg.temp_staging_raw_audit_jenkins FROM 's3://dev-cucumbers/FeedIngestionStatusAudit/jenkins/{filename}' FILE_FORMAT = ( TYPE='JSON' DATE_FORMAT='YYYY-MM-DD' TIMESTAMP_FORMAT=AUTO COMPRESSION=AUTO STRIP_OUTER_ARRAY=TRUE ) FORCE=TRUE MATCH_BY_COLUMN_NAME = CASE_INSENSITIVE CREDENTIALS=( AWS_KEY_ID='{os.environ.get("DEV_AWS_ACCESS_KEY_ID")}' AWS_SECRET_KEY='{os.environ.get("DEV_AWS_SECRET_ACCESS_KEY")}' AWS_TOKEN='{os.environ.get("DEV_AWS_SESSION_TOKEN")}' ); """ merge_into_query = """ MERGE INTO dev_engineering.sduberg.staging_raw_audit_jenkins as target USING dev_engineering.sduberg.temp_staging_raw_audit_jenkins as source ON target.job = source.job AND target.build_id = source.build_id WHEN MATCHED THEN UPDATE SET target.status = source.status, target.parameters = source.parameters, target.start_time = source.start_time, target.end_time = source.end_time WHEN NOT MATCHED THEN INSERT ( job, build_id, status, parameters, start_time, end_time ) VALUES ( source.job, source.build_id, source.status, source.parameters, source.start_time, source.end_time ) """ snow.conn.cursor().execute(truncate_staging_query) snow.conn.cursor().execute(copy_into_query) snow.conn.cursor().execute(merge_into_query) if __name__ == "__main__": jenkins()