"""Github Flow. Contains functions and variables necessary to work with github and sync over files between instances. """ import datetime import re import github import config GIT_INSTANCE = github.Github(config.GITHUB_TOKEN) MAIN_REPO = GIT_INSTANCE.get_repo('theorchard/looker-orchard-analytics') # Change this to your user FORKED_CLIENT_REPO =\ GIT_INSTANCE.get_repo('rolkhovskiy/looker-client-orchard-analytics') CLIENT_ORIGIN_REPO =\ GIT_INSTANCE.get_repo('theorchard/looker-client-orchard-analytics') PREFIX_FOR_REPLACING_ACCESS_FILTER = r'access_filter:' def post_file(filename, content): """Create a file on client instance repo. Pre-condition: a file with given filename should NOT already exist in a client repo. Otherwise, refer to update_file_git function. Function should not be renamed to create_file to avoid name clashing with github repo class methods. Args: filename (str): name of the file. Comes with full extension. Example -> core_metadata.model.lkml content (str): string representation of a file content. Comes as a return value of get_file_contents. """ FORKED_CLIENT_REPO.create_file( filename, # file name f'{datetime.date.today()} {filename} sync', # Commit message content, # File content branch='master' # branch ) def update_file_git(filename, content): """Update a file on a client repo with content from main repo. Pre-condition: a file with given filename should already exist in a client repo. Otherwise, refer to post_file function. Function should not be renamed to update_file to avoid name clashing with methods of github repo class methods. Args: filename (str): name of the file. Comes with full extension. Example -> core_metadata.model.lkml content (str): string representation of a file content. Comes as a return value of get_file_contents. """ existing_file = FORKED_CLIENT_REPO.get_contents(filename) FORKED_CLIENT_REPO.update_file( path=existing_file.path, message=f'{datetime.date.today()} {filename} sync', content=content, sha=existing_file.sha, branch='master' ) def create_pull_request(): """Create pull request from forked master to client master . Head should later be changed to appropriate one instead of rolkhovskiy:master. """ CLIENT_ORIGIN_REPO.create_pull( title=f'Synchronization: {datetime.date.today()}', # Title body='Alignment', # Description base='master', # Merged into head='rolkhovskiy:master', # Taken from ) def change_blocks_in_model_code(prefix, replace_with, model_file_content): """Regex function to adjust blocks of code in model file. This function performs complex regular expression which consists of few parts and returns the end result of a regex. Args: prefix (str): part of a regex pattern. Should only use either of two consts declared in the beginning of this module. This determines which block of code will be altered. As of time of writing this, there are only two cases for such an alteration: access_filter block of code and a join block for access filter. Thus there are only two available prefixes for now. replace_with (str): part of a regex pattern. In case of removing access filter join this should be an empty string. In case of replacing access filter itself with another one suitable for client instance this should come as in input variable new_access_filter. model_file_content (str): string representation of a model file contents. Comes as a return value of get_model_file_contents function. Returns: str: model_file_content with applied regex to it. """ blocks_regex = r'\s?\{(?:\s|\S(?!\n\s+\}))*\S\s+\}' replacement = r'{}{}'.format(prefix, blocks_regex) return re.sub(replacement, replace_with, model_file_content) def change_connection_in_model_file(file_contents, correct_connection): """Change the name of a DB connection within a model file. Args: file_contents (str): string representation of model file content. correct_connection (str): correct DB connection for client instance models. Returns: str: file_contents with correct DB connection. """ return re.sub( r'snowflake_via_lookerappadmin', correct_connection, file_contents) def get_prefix_for_removing_join_access_filter(filter_string_body): """Return correct regex for removing access filter join. Args: filter_string_body (str): """ return ''.join([r'\s{4}join:\s?', filter_string_body]) def edit_model_file( original_model_file_content, new_access_filter, remove_access_filter_join, connection ): """Edit model file using regex(es). First it removes a block of code for join access filter, if such block exists. Then, it changes the contains of access_filter block to a correct one for client instance. The correct one comes as an input. Finally, it changes the connection to a correct one (see change_connection_in_model_file function). Args: original_model_file_content (str): string representation of a model file contents. Comes as a return value of get_model_file_contents function. new_access_filter (str): one-line representation of new access filter code block.. remove_access_filter_join (str): name of the join which code block should be removed. connection (str): client instance db connection name. Returns: str: string representation of a model file content, with all alterations applied, client instance compliant. """ prefix_for_removing_join_access_filter =\ get_prefix_for_removing_join_access_filter(remove_access_filter_join) data = change_blocks_in_model_code( prefix_for_removing_join_access_filter, '', original_model_file_content) data = change_blocks_in_model_code( PREFIX_FOR_REPLACING_ACCESS_FILTER, new_access_filter, data) data = change_connection_in_model_file( data, connection) return data def check_file_in_repo(repo, file_name): """Check if the given file exists in the given repo. Args: repo: PyGithub instance of a repo. Should only use *_REPO constants for this argument. file_name (str): file name of the file to check, with full extension. Returns: str: string representation of a file contents in readable format. bool: False if file doesn't exist. """ try: contents = repo.get_contents(file_name) return contents.decoded_content.decode() except github.UnknownObjectException: return False def check_file_in_client_repo(file_name): """Call check_file_in_repo for client repo. Args: file_name (str): name of the file with full extension. Returns: check_file_in_repo call. """ return check_file_in_repo(FORKED_CLIENT_REPO, file_name) def sync_git_model( model_file_name, new_access_filter, remove_access_filter_join, connection): """Sync .model.lkml file with proper adjustments. Gets original content through editing functions to change connection and joins. Args: model_file_name (str): file name with full extension. Example -> core_metadata.model.lkml new_access_filter (str): one-line representation of new access filter code block for .model.lkml file. remove_access_filter_join (str): name of the join which code block should be removed from .model.lkml file. connection (str): client instance db connection name. Returns: bool: False if model wasn't found in main repo. True if successfully created / updated a model. """ original_content = check_file_in_main_repo(model_file_name) if original_content: client_ready_content = edit_model_file( original_content, new_access_filter, remove_access_filter_join, connection) client_model_content = check_file_in_client_repo(model_file_name) if client_ready_content == client_model_content: return True if client_model_content: update_file_git(model_file_name, client_ready_content) return True else: post_file(model_file_name, client_ready_content) return True return False def check_file_in_main_repo(file_name): """Call check_file_in_repo for main repo. Args: file_name (str): name of the file with full extension. Returns: check_file_in_repo call. """ return check_file_in_repo(MAIN_REPO, file_name) def check_for_absent_view_files(list_of_file_names): """Check if all provided files are present in the main instance repo. This function is more of a sanity check. Previously there was a problem when files in the repo contained periods in their file names (besides extensions) because looker API would only allow underscores. It resulted in the fact that github API received a list of files to sync over, but couldn't find many of such files if they contained periods (or other symbols for that sake). Since then there is a convention of only using single underscores (and not periods) in file names in the main repo. Args: list_of_file_names(list, tuple or set): iterable container with file names to check in the main repo. Comes as a return value of get_all_view_files function from model_tools.py module. Returns: set: if everything is ok and all provided file names correspond to files in the main repo this set is going to be empty, thus evaluating to False. This is a desirable outcome of this function call. If the return set is not empty -> it contains names of files which should be checked. Those are probably absent in the main repo or more likely named incorrectly. Look for periods (besides file extension) and double (triple?) underscores. """ absent_files = set() for filename in list_of_file_names: full_file_name = f'{filename}.view.lkml' if not check_file_in_main_repo(full_file_name): absent_files.add(f'{filename}') return absent_files def sync_view_files(list_of_file_names): """Sync view files from main repo to client repo. Should only run this once check_for_absent_files with the same input evaluated to False. Refer to check_for_absent_files documentation Returns section for detailed explanation. With each given file name: if such a file already exists in the client repo and it's content is the same as the main file content - nothing happens. If the two file contents are different - client file is updated. Otherwise it will create a new file. Args: list_of_file_names (set): iterable (list, tuple, set) containing strings. Strings are file names. """ for filename in list_of_file_names: name = f'{filename}.view.lkml' main_content = check_file_in_main_repo(name) client_content = check_file_in_client_repo(name) if main_content == client_content: continue if client_content: update_file_git(name, main_content) else: post_file(name, main_content) def complete_views_sync(set_of_file_names): """Sync view files in bulk. Checks for absent files in a given list of file names. If there are - prints the list. Of there are no absent files - syncs all of them. Updates existing on client instance and creates new if there is no existing file for a file name. Args: set_of_file_names (set): iterable (list, tuple, set) containing strings. Strings are file names. Returns: bool: True if synced successfully, False if there were absent files. """ absent_files = check_for_absent_view_files(set_of_file_names) if not absent_files: sync_view_files(set_of_file_names) return True else: print(f'Absent files:') for name in absent_files: print(f' - {name}') return False