import datetime import time from garcon import task import config import utils import s3 @task.decorate(timeout=60) def print_hello_task(activity, workflow_id, activity_name, task_name, sleep): """Say hello! Args: activity (ActivityWorker): Garcon ActivityWorker executing the task. workflow_id (str): SWF workflowid executing the task. activity_name (str): Name identifying the activity executing the task. task_name (str): Name identifying the task. sleep (str): Optional time in seconds to sleep before starting task. Returns: dict: Context with (str) timestamp of when the task completed. """ if sleep: time.sleep(sleep) message = '{workflow_id}: I am {activity_name}, {task_name}'.format( workflow_id=workflow_id, activity_name=activity_name, task_name=task_name) print(message) return {'hello_completed': str(datetime.datetime.now())} @task.decorate(timeout=60) def bootstrap(activity, content_owner, job_raw, starttime): """ checks to see if a s3 file (the target) exists for a given report type / CMS/ date. a response body is returned with metadata about the report. """ endtime = starttime # add one day! print(content_owner) job_name = job_raw + config.string_test # normalize names for v1.1 reports # job_name = config.v1_reports.get(job_name, job_name) response = { "job_name": job_name, "starttime": starttime, "endtime": endtime } return response @task.decorate(timeout=60) def request_report(): """ checks if the file is available from Youtube API """ if utils.exists_upstream() == False: return False else: return True @task.decorate(timeout=60) def report_to_disk(): if not utils.exists_upstream() or utils.exists_in_s3(): return return utils.download_report() @task.decorate(timeout=60) def local_to_s3(): if not os.path.isfile() or utils.exists_in_s3(): return return utils.upload_report() @task.decorate(timeout=60) def s3_to_snowflake(): if not utils.exists_in_s3() or utils.exists_in_table(): return return utils.insert_to_table()