import os from datetime import datetime, timedelta ENV = 'PROD' home = os.environ.get('AIRFLOW_HOME') # Airflow Stuff if ENV == 'DEV': dag_id = 'dev_yt_reports' else: dag_id = 'yt_reports' cron_schedule = '00 8 * * *' # Midnight in PDT, 3 AM in EST start_date = datetime(2017, 2, 22) retry_delay = timedelta(minutes=2) pool = 'medium_pool' # where is pigz installed? if os.environ.get('HOME') == '/home/lyin': pigz = '/home/lyin/.linuxbrew/bin/pigz' else: pigz = 'pigz' # Dictionaries and definitions for YT API v1_reports = { 'basica3': 'useractivity', 'assetestimatedrevenuea1': 'assetrevenue', 'estimatedrevenuea1' : 'revenue', 'adratesa1': 'adrates' } ORCH = 'J8vAyKuNSYBIN_9RIdxggQ' ENT = 'oVAQ96fyMa57I6WozL2gUw' IODA = 'n293L2XLLmtg5wlKBao5Uw' CMS = [ORCH,ENT,IODA] CMS_DICT = { ORCH:'ORCH', ENT:'ENT', IODA:'IODA' } # these reports (job_name from jobs_no_playlist.json) need to be backfilled. # note those reports are run through utils.get_job_name BACKFILL = [ 'useractivity', 'assetbasic', 'estimatedrevenue', 'assetestimatedrevenue' ] # Credentials CREDS = os.environ.get('CRED_PATH') STAGE = os.environ.get('YT_STAGE') if CREDS: CLIENT_SECRETS_FILE = os.path.join(CREDS,'client_secrets_installed.json') TOKEN = os.path.join(CREDS,'youtube_access_token.json') # S3 locations BUCKET = 's3://prod-datalytics' if ENV == 'DEV': KEY = 'etl/store/youtube/bulk_reports_api_dev' else: KEY = 'etl/store/youtube/bulk_reports_api' S3_PATH = os.path.join(BUCKET,KEY) # Snowflake stuff if ENV != 'DEV': PROD_DB = 'PROD' PROD_SCHEMA = 'YT_REPORTS_API' else: PROD_DB = 'LY_DEV' PROD_SCHEMA = 'YOUTUBE_TEST' PROD_FMT = 'LY_DEV.TEMPLATES.YOUTUBE_DAILY' WH = 'YOUTUBE_ETL' # Youtube Stuff job_meta = os.path.join( home, "dags/projects/youtube_reports_api/data_in/jobs_no_playlists.json" ) # ANALYTICS API Versioning YOUTUBE_REPORTING_API_SERVICE_NAME = "youtubereporting" YOUTUBE_REPORTING_API_VERSION = "v1" # Scopes needed to get Authenticated YOUTUBE_REPORT_SCOPES = ( "https://www.googleapis.com/auth/youtube.readonly", "https://www.googleapis.com/auth/yt-analytics.readonly", "https://www.googleapis.com/auth/yt-analytics-monetary.readonly", "https://www.googleapis.com/auth/youtubepartner", "https://www.googleapis.com/auth/youtubepartner-channel-audit" ) MISSING_CLIENT_SECRETS_MESSAGE = """ WARNING: Please configure OAuth 2.0 To make this sample run you will need to populate the client_secrets.json file found at: %s with information from the Developers Console https://console.developers.google.com/ For more information about the client_secrets.json file format, please visit: https://developers.google.com/api-client-library/python/guide/aaa_client_secrets """ % os.path.abspath(CLIENT_SECRETS_FILE)