"""Geocoding Workflow. ================== A Garcon workflow for updating geographic information in dim_zip table. Workflow can be run by the folloing exec command: For just one country: dim-refresh geocoding -c '{"country_code":"NL"}' exec dim_zip. For all country dim-refresh geocoding -c exec dim_zip. """ import logging from garcon import runner from garcon.param import StaticParam from garcon_contrib.aws import garcon_s3 from dim_refresh_etl.flows.base import BaseFlow from dim_refresh_etl.flows.geocoding import settings from dim_refresh_etl.flows.geocoding import tasks logger = logging.getLogger('dim_refresh_etl') class Flow(BaseFlow): """Geocoding flow class.""" timeout = 21600 # 6 hours (60 *60 * 6) def __init__(self): """Create a Geocoding Workflow.""" super().__init__(name='geocoding', version='1.0') def decider(self, schedule, context): """Flow decider. Arg: schedule (callable): Call the scheduler. context (dict): initial context workflow was launched with. """ write_lat_long = schedule( 'write_lat_long', self.write_country_zip_code_lat_long_map) schedule( 'update_lat_long_on_dim_zip', self.update_lat_long_on_dim_zip, requires=[write_lat_long]) def workflow_id(self, context): """Generate the workflow ID.""" return 'geocoding' @property def write_country_zip_code_lat_long_map(self): """Write country ZIP code lat_long map.""" return self.create( name='write_lat_log', tasks=runner.Sync( garcon_s3.remove_files_from_path.fill( namespace='delete_old_s3key', path=StaticParam(settings.TEMP_S3_PATH), return_deleted_files=StaticParam(False)), tasks.write_country_zip_code_lat_long_map.fill( namespace='write_country_zip_code_lat_long_map', country_code='country_code', sfdb_params='sfdb_params'))) @property def update_lat_long_on_dim_zip(self): """Update lat_long file on dim_zip.""" return self.create( name='update_lat_long_on_dim_zip', tasks=runner.Sync( tasks.update_lat_long_on_dim_zip.fill( namespace='update_lat_long_on_dim_zip', sfdb_params='sfdb_params')))