"""Generators for activities of the sql2sf workflow.""" from snowflake_etl.flows.sql2sf import config from snowflake_etl.flows.sql2sf import helpers def _get_min_max_pk(primary_key, source_schema, source_table, mysqldb_config): """Get min and max primary key values from a source table table. Args: primary_key (str): Primary key of table from which data is retrieved. source_schema (str): The name of the source schema (e.g., 'production'. source_table (str): The name of the source table (e.g., 'contact'. mysqldb_config (tuple): Host, port, username and password for a db. Returns: tuple: Min and max values of primary key. """ sql = """ SELECT MIN({primary_key}) as min_id, MAX({primary_key}) as max_id FROM `{source_schema}`.`{source_table}` """.format( source_schema=source_schema, primary_key=primary_key, source_table=source_table) results = helpers.execute_with_mysql(sql, 'fetchone', *mysqldb_config) min_id, max_id = results return min_id, max_id def get_columns(source_schema, source_table, mysqldb_config): """Return a list of column names. Args: source_schema (str): The name of the source schema (e.g., 'production'. source_table (str): The name of the source table (e.g., 'contact'. mysqldb_config (tuple): Host, port, username and password for a db. Returns: list: A list of column names. """ types_to_wrap = { 'char': True, 'varchar': True, 'text': True, 'mediumtext': True, 'longtext': True, 'enum': True, 'set': True} sql = """SELECT COLUMN_NAME, DATA_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA = '{source_schema}' AND TABLE_NAME = '{source_table}' ORDER BY ORDINAL_POSITION;""".format( source_schema=source_schema, source_table=source_table) results = helpers.execute_with_mysql(sql, 'fetchall', *mysqldb_config) columns = [(column[0], types_to_wrap.get(column[1])) for column in results] return columns def get_min_max_pairs( primary_key, chunk_size, source_schema, source_table, mysqldb_config): """Return a list of min_id and max_id pairs for each data chunk. Args: primary_key (str): Primary key of table from which data is retrieved. chunk_size (int): Chunk size from entity setting. source_schema (str): The name of the source schema (e.g., 'production'. source_table (str): The name of the source table (e.g., 'contact'. mysqldb_config (tuple): Host, port, username and password for a db. Returns: list: A list of dictionaries of min_max pairs. """ min_id, max_id = _get_min_max_pk( primary_key, source_schema, source_table, mysqldb_config) pairs = [ { 'min_id': pk, 'max_id': min(pk + chunk_size - 1, max_id) } for pk in range(min_id, max_id, chunk_size)] return pairs def unload_mysql_data_params(context): """Prepare input params for the unload_mysql_data task. Args: context (dict): Dictionary which is being passed through the workflow. Yields: dict: min_id and max_id dictionary for being iterated through. """ source_db_host = context.get('bootstrap_sql2sf.source_db_host') source_schema = context.get('bootstrap_sql2sf.source_schema') source_table = context.get('bootstrap_sql2sf.source_table') sources = config.SourcesConf() mysqldb_config = sources.get_db_credentials(source_db_host, source_schema) columns = get_columns(source_schema, source_table, mysqldb_config) if sources.is_unload_in_chunks( source_db_host, source_schema, source_table): chunk_size = sources.get_chunk_size( source_db_host, source_schema, source_table) primary_key = sources.get_primary_key( source_db_host, source_schema, source_table) min_max_pairs = get_min_max_pairs( primary_key, chunk_size, source_schema, source_table, mysqldb_config) for pair in min_max_pairs: yield {'generator.table': source_table, 'generator.columns': columns, 'generator.bucket': context.get( 'bootstrap_sql2sf.destination_s3_bucket'), 'generator.destination_s3_key': context.get( 'bootstrap_sql2sf.destination_s3_key'), 'generator.min_id': pair.get('min_id'), 'generator.max_id': pair.get('max_id')} else: yield { 'generator.table': source_table, 'generator.columns': columns, 'generator.bucket': context.get( 'bootstrap_sql2sf.destination_s3_bucket'), 'generator.destination_s3_key': context.get( 'bootstrap_sql2sf.destination_s3_key')}