"""Migrate users to auth0.""" from collections import defaultdict import json from typing import Dict, List, Tuple import plyvel import auth0 import config from logger import logger import sql import utils def get_db_users_roles() -> Tuple[List[dict], Dict[str, List[str]]]: """Get list of users and roles from levelDB. Returns: Tuple[List[dict], Dict[str, List[str]], List[str]]: List of users and roles to user IDs mapping. """ db = plyvel.DB(config.DB_PATH, create_if_missing=False) users = [] roles = defaultdict(list) with open(config.Files.DB_DUMP_TXT, 'w') as db_dump: for key, value in db.iterator(start=b'adminaccounts~', stop=b'adminaccounts~~'): value_str = value.decode("utf-8") db_dump.write(value_str + '\n') data = json.loads(value_str) user_id = data['id'] users.append({ 'user_id': user_id, 'email': data['email'], 'name': data['name'], 'email_verified': True, 'blocked': data['suspended'], 'user_metadata': {'apollo': {'homeMarketID': data['homeMarketID']}}, 'app_metadata': {'marketPermissions': data['marketPermissions']} }) roles['Apollo' + data['role'].title()].append(user_id) db.close() return users, dict(roles) def get_users_roles() -> Tuple[List[dict], Dict[str, List[str]]]: """Get list of users and roles from files or levelDB. Returns: Tuple[List[dict], Dict[str, List[str]], List[str]]: List of users and roles to user IDs mapping. """ utils.check_dir(config.Files.DIR) if utils.check_files_exist(*config.Files.ALL_JSON): users, roles = utils.read_json_files(*config.Files.ALL_JSON) else: users, roles = get_db_users_roles() utils.write_json_files(**{ config.Files.USERS: users, config.Files.ROLES: roles }) return users, roles def index(): """Run export/import users and roles. """ logger.info('Getting users and roles.') users, roles = get_users_roles() logger.info('Got {} users.'.format(len(users))) if config.IMPORT_USER_IDS or config.IGNORE_USER_IDS: users = [ u for u in users if ( (not config.IMPORT_USER_IDS or u['user_id'] in config.IMPORT_USER_IDS) and (not config.IGNORE_USER_IDS or u['user_id'] not in config.IGNORE_USER_IDS) ) ] filtered_roles = {} for role, user_ids in roles.items(): user_ids = [ user_id for user_id in user_ids if ( (not config.IMPORT_USER_IDS or user_id in config.IMPORT_USER_IDS) and (not config.IGNORE_USER_IDS or user_id not in config.IGNORE_USER_IDS) ) ] if user_ids: filtered_roles[role] = user_ids roles = filtered_roles if config.Action.DELETE_INCORRECT_ID_RECORDS: existing_users = auth0.export_users() logger.info('Delete users that exist in levelDB and Auth0 with another ID.') for user in users: user_id = user['user_id'] email = user['email'] if email in existing_users and user_id != existing_users[email]: logger.info( 'Removing user {} as current user ID {} is incorrect'.format( user_id, existing_users[email])) auth0.delete_user(user_id) if config.Action.DELETE_EXISTING_RECORDS: existing_users = auth0.export_users() logger.info('Delete users that exist in Auth0 AND levelDB.') for user in users: email = user['email'] if email in existing_users: user_id = existing_users[email] logger.info('Removing user {}'.format(user_id)) auth0.delete_user(user_id) roles_mapping = auth0.get_all_roles() logger.info('Delete roles that exist in Auth0 AND levelDB.') for role in roles.keys(): if role in roles_mapping: logger.info('Removing role {}.'.format(role)) auth0.delete_role(roles_mapping[role]) if config.Action.SAVE_MARKETS: user_market = [ (u['user_id'], u['user_metadata']['apollo']['homeMarketID']) for u in users] logger.info('Save user to home market ID relations into MySQL.') sql.insert_user_market(user_market) if config.Action.SAVE_USERS: logger.info('Import users to Auth0.') failed_users = auth0.import_users(users) # filter failed users if failed_users: roles = { role: [user_id for user_id in user_ids if user_id not in failed_users] for role, user_ids in roles.items() } if config.Action.SAVE_ROLES: logger.info('Import users roles to Auth0.') for role, user_ids in roles.items(): role_id = auth0.get_role(role) if not role_id: logger.info('Creating role "{}".'.format(role)) role_id = auth0.create_role(role) logger.info('Assign users to role "{}".'.format(role)) auth0.assign_users_role(user_ids, role_id) if config.Action.GET_RESULTS_AND_COMPARE: logger.info('Exporting data for testing.') logger.info('Loading Auth0 users data.') auth0_users = auth0.export_users( [ 'user_id', 'email', 'name', 'blocked', 'user_metadata.apollo.homeMarketID', 'app_metadata.marketPermissions' ]) logger.info('Loading roles and assignments.') auth0_roles = auth0.get_all_roles() users_roles = {} for role_name, role_id in auth0_roles.items(): role_users = set(auth0.get_role_assignments(role_id)) for user_id in role_users: if user_id in users_roles: logger.error('More than one role for user {} found.'.format(user_id)) if users_roles[user_id] in ('user', 'superuser'): continue users_roles[user_id] = role_name logger.info('Saving Auth0 users to CSV.') with open(config.Files.AUTH0_DUMP_CSV, 'w') as auth0_csv: users_data = auth0_users.values() for user in users_data: if user['user_id'] in users_roles: user['role'] = users_roles[user['user_id']] auth0_csv.write('\t'.join([str(f) for f in user.values()]) + '\n') is_valid = True logger.info('Saving LevelDB users to CSV.') with open(config.Files.DB_DUMP_CSV, 'w') as db_csv: for user in users: user_role = ','.join([r for r, u in roles.items() if user['user_id'] in u]) user_data = [ user['user_id'], user['email'], user['name'], str(user['blocked']), str(user['user_metadata']['apollo']['homeMarketID']), json.dumps(user['app_metadata']['marketPermissions']), user_role, ] if user['email'] not in auth0_users: logger.error('User {} was not imported.'.format(user['user_id'])) is_valid = False else: auth0_user = auth0_users[user['email']] if (auth0_user['user_id'] != user['user_id'] or auth0_user['name'] != user['name'] or auth0_user['blocked'] != user['blocked'] or auth0_user['user_metadata.apollo.homeMarketID'] != ( user['user_metadata']['apollo']['homeMarketID']) or auth0_user['app_metadata.marketPermissions'] != ( user['app_metadata']['marketPermissions']) or ('role' not in auth0_user or auth0_user['role'] != user_role)): logger.error('User {} data is not valid.'.format(user['user_id'])) logger.info(user) logger.info(auth0_user) is_valid = False db_csv.write('\t'.join(user_data) + '\n') if is_valid: logger.info('Data check - OK') if __name__ == '__main__': index()