"""Pipelines. Contains "one button" high-level functions for syncing things over. """ import config import dashboard_tools import github_tools import look_tools import model_tools import query_tools def sync_look(main_look_id): """Sync over a look. Get's original look id and performs all the necessary operations (by calling other functions) for look to go to client instance, given that look with such id exists. Finally, it checks by title if such look already exists on client instance or not. If not - it just creates a new look on client instance; otherwise - it updates already existing look. Update will also sync over the query, creating a new one. Args: main_look_id (int): id of a look on main looker instance to sync over. Found in looks URL after .com/looks/ Returns: bool: result of an action or False if look wasn't found. """ main_look_json = look_tools.get_look_json(main_look_id) if main_look_json: client_query_id = query_tools.sync_query(main_look_json) client_look_json =\ look_tools.adjust_look_data(main_look_json, client_query_id) client_check_id =\ look_tools.check_look_on_client_instance(main_look_json) if client_check_id: return look_tools.update_look(client_check_id, client_look_json) else: return look_tools.post_look_to_client_instance(client_look_json) return False def git_files_model_sync( model_name, model_json, new_access_filter, remove_access_filter_join, connection): """Sync all the git-related files for a given model. First it syncs over .model.lkml file, then it syncs all the joined view files for all the explores of a given model and creates a pull request. Args: model_name (str): name of the model in snake_case without file extension. model_json (dict): json representation of a model. new_access_filter (str): one-line representation of new access filter code block for .model.lkml file. remove_access_filter_join (str): name of the join which code block should be removed from .model.lkml file. connection (str): client instance db connection name. Returns: bool: True if sync was successful, False otherwise. """ model_file_name = f'{model_name}.model.lkml' model_file_synced = github_tools.sync_git_model( model_file_name, new_access_filter, remove_access_filter_join, connection) if model_file_synced: set_of_explore_names = model_tools.get_set_of_explore_names(model_json) for explore_name in set_of_explore_names: set_of_view_files = model_tools.get_set_of_views_for_explore( model_name, explore_name) github_tools.complete_views_sync(set_of_view_files) return True return False def full_sync_model(input_json): """Complete model sync including looker instance and github files. Args: input_json (dict): dict with inputs. Contains name, filter, join to remove, connection name. Returns: bool: True if sync was successful, False otherwise. """ name = input_json.get('name') new_access_filter = input_json.get('new_access_filter') remove_access_filter_join = input_json.get('remove_access_filter_join') connection = input_json.get('connection') model_json = model_tools.get_main_instance_model_json(name) if model_json: files_synced = git_files_model_sync( name, model_json, new_access_filter, remove_access_filter_join, connection) if files_synced: lookml_synced =\ model_tools.sync_lookml_model(name, model_json) return lookml_synced return False return False def sync_dashboard_element(main_instance_element_json, client_dashboard_id): """Sync dashboard element over client looker instance. Args: main_instance_element_json (dict): json of a dashboard element. client_dashboard_id (int): id of dashboard on a client looker instance. Returns: calls either post or update function depending on element_title argument. """ client_query_id = None client_look_id = None query_id = main_instance_element_json.get('query_id') if query_id: client_query_id = query_tools.sync_query(main_instance_element_json) look_id = main_instance_element_json.get('look_id') if look_id: client_look_id = sync_look(look_id) client_ready_json = dashboard_tools.adjust_dashboard_element_json( main_instance_element_json, client_dashboard_id, client_query_id, client_look_id) return dashboard_tools.post_dashboard_element(client_ready_json) def full_sync_dashboard(main_dashboard_id): """Sync dashboard to the client looker instance. It checks if a dashboard with a title of a given dashboard already exists on client instance. If it does - the dashboard will be updated, if not - it will be created. Then the same process goes for dashboard filters. Then for each layout the dashboard has - it is updated and all dashboard elements and layout components are built from scratch. Args: main_dashboard_id (int): id of a dashboard on main looker instance. """ original_dashboard_json =\ dashboard_tools.get_dashboard_json(main_dashboard_id) client_ready_dashboard_json =\ dashboard_tools.adjust_dashboard_json(original_dashboard_json) dashboard_title = original_dashboard_json.get('title') original_filters_list = original_dashboard_json.get('dashboard_filters') client_dashboard_id =\ dashboard_tools.check_dashboard_on_client_instance(dashboard_title) if not client_dashboard_id: client_dashboard_json =\ dashboard_tools.post_dashboard(client_ready_dashboard_json) client_dashboard_id = client_dashboard_json.get('id') dashboard_tools.sync_dashboard_filters( original_filters_list, client_dashboard_id) else: client_dashboard_json = dashboard_tools.update_client_dashboard( client_ready_dashboard_json, client_dashboard_id) client_list_of_filters =\ dashboard_tools.get_list_of_dashboard_filters(client_dashboard_id) dashboard_tools.sync_dashboard_filters( original_filters_list, client_dashboard_id, client_list_of_filters) main_list_of_layouts = original_dashboard_json.get('dashboard_layouts') client_list_of_layouts = client_dashboard_json.get('dashboard_layouts') client_list_of_elements = client_dashboard_json.get( 'dashboard_elements', []) client_list_of_element_ids =\ (element['id'] for element in client_list_of_elements) for element_id in client_list_of_element_ids: dashboard_tools.delete_dashboard_element(element_id) for client_layout in client_list_of_layouts: for main_layout in main_list_of_layouts: if client_list_of_layouts.index(client_layout) ==\ main_list_of_layouts.index(main_layout): client_layout_id = client_layout.get('id') main_layout['dashboard_id'] = client_dashboard_id dashboard_tools.update_dashboard_layout( client_layout_id, main_layout) main_layout_id = main_layout.get('id') main_list_of_layout_components = dashboard_tools.\ get_all_dashboard_layout_components( main_layout_id) for component in main_list_of_layout_components: main_element_id = component.get('dashboard_element_id') main_element_json = dashboard_tools.\ get_dashboard_element_json(main_element_id) new_ele_id = sync_dashboard_element( main_element_json, client_dashboard_id) component['dashboard_layout_id'] = client_layout_id component['dashboard_element_id'] = new_ele_id recent_list_of_components = dashboard_tools.\ get_all_dashboard_layout_components( client_layout_id, config.LOOKER_CLIENT_INSTANCE_URL) recent_layout_component_id =\ recent_list_of_components[-1]['id'] dashboard_tools.update_layout_component( component, recent_layout_component_id)