"""Tests for sql2sf config, especially for SourcesConf class.""" import pytest def test_init(sources_mock): """Test __init__ method of the SourcesConf class.""" assert isinstance(sources_mock.conf, dict) assert set(sources_mock.conf) == {'s3_destinations', 'queries', 'db_hosts'} def test_get_tables_to_sync(sources_mock): """Test get_tables_to_sync method.""" assert len(sources_mock.get_tables_to_sync()) == 5 assert set(sources_mock.get_tables_to_sync()) == { ('research_collections', 'ppb_collections', 'track_delivery_history_7', "FIELD_DELIMITER='\t' " "RECORD_DELIMITER='\n'"), ('ows_rds', 'sales_goals', 'sales_goal', None), ('ows_rds', 'youtube_ownership', 'yt_api_log', None), ('redshift04', 'production', 'dim_artist', None), ('redshift04', 'production', 'fact_analytics', None)} def test_get_db_credentials(sources_mock): """Test get_db_credentials method.""" assert isinstance(sources_mock.get_db_credentials( 'ows_rds', 'sales_goals'), tuple) assert len(sources_mock.get_db_credentials( 'ows_rds', 'sales_goals')) == 4 with pytest.raises(KeyError): sources_mock.get_db_credentials( 'ows_rds', 'nonexistent_schema') with pytest.raises(KeyError): sources_mock.get_db_credentials( 'nonexistent_db', 'sales_goals') def test_get_db_type(sources_mock): """Test get_db_type method.""" assert sources_mock.get_db_type('ows_rds') == 'mysql' assert sources_mock.get_db_type('redshift04') == 'redshift' with pytest.raises(KeyError): sources_mock.get_db_type('nonexistent db') def test_get_table_params(sources_mock): """Test get_table_params method.""" assert sources_mock.get_table_params( 'ows_rds', 'youtube_ownership', 'yt_api_log') == { 'sanity_threshold': 0, 'chunk_size': 20000000, 'primary_key': 'id', 'unload_in_chunks': True, 'wrap_to_stripspecialchars': True} with pytest.raises(KeyError): sources_mock.get_table_params( 'ows_rds', 'youtube_ownership', 'nonexistent_table') def test_get_table_props_by_name(sources_mock): """Test get_table_props_by_name method.""" assert sources_mock.get_table_props_by_name('yt_api_log') == ( 'ows_rds', 'youtube_ownership', 'yt_api_log', None) with pytest.raises(KeyError): sources_mock.get_table_props_by_name('nonexistent_table') def test_get_query(sources_mock): """Test qet_query method.""" assert 'SELECT' in sources_mock.get_query('mysql', 'snapshot') assert 'BETWEEN' in sources_mock.get_query('redshift', 'date_range') with pytest.raises(KeyError): sources_mock.get_query('sqlite', 'snap_range') def test_is_unload_in_chunks(sources_mock): """Test unload_in_chunks method.""" assert sources_mock.is_unload_in_chunks( 'ows_rds', 'sales_goals', 'sales_goal') is False assert sources_mock.is_unload_in_chunks( 'ows_rds', 'youtube_ownership', 'yt_api_log') with pytest.raises(KeyError): sources_mock.is_unload_in_chunks( 'ows_rds', 'youtube_ownership', 'nonexistent_table') def test_get_chunk_size(sources_mock): """Test get_chunk_size method.""" assert not sources_mock.get_chunk_size( 'ows_rds', 'sales_goals', 'sales_goal') assert (sources_mock.get_chunk_size( 'ows_rds', 'youtube_ownership', 'yt_api_log') == 20000000) with pytest.raises(KeyError): sources_mock.get_chunk_size( 'ows_rds', 'youtube_ownership', 'nonexistent_table') def test_get_primary_key(sources_mock): """Test get_primary_key method.""" assert not sources_mock.get_primary_key( 'ows_rds', 'sales_goals', 'sales_goal') assert (sources_mock.get_primary_key( 'ows_rds', 'youtube_ownership', 'yt_api_log') == 'id') with pytest.raises(KeyError): sources_mock.get_primary_key( 'ows_rds', 'youtube_ownership', 'nonexistent_table') def test_get_s3_path_data(sources_mock): """Test get_s3_path_data method.""" assert 's3://{env}-db-data/sql2sf/' in sources_mock.get_s3_path_data( 'mysql') assert 's3://{env}-db-data/redshift/' in sources_mock.get_s3_path_data( 'redshift') with pytest.raises(KeyError): sources_mock.get_s3_path_data('nonexistent_db') def test_get_s3_path_schema(sources_mock): """Test get_s3_path_schema method.""" assert 's3://{env}-db-data/sql2sf/' in sources_mock.get_s3_path_schema( 'mysql') assert 's3://{env}-db-data/redshift/' in sources_mock.get_s3_path_schema( 'redshift') with pytest.raises(KeyError): sources_mock.get_s3_path_schema('nonexistent_db') def test_is_table_supports_incremental_sync(sources_mock): """Test test_is_table_supports_incremental_sync method.""" assert not sources_mock.is_table_supports_incremental_sync( 'ows_rds', 'youtube_ownership', 'yt_api_log') assert sources_mock.is_table_supports_incremental_sync( 'redshift04', 'production', 'fact_analytics') with pytest.raises(KeyError): sources_mock.is_table_supports_incremental_sync( 'nonexistent_db', 'production', 'fact_analytics') def test_get_sync_sanity_threshold(sources_mock): """Test get_sync_sanity_threshold method.""" assert sources_mock.get_sync_sanity_threshold( 'ows_rds', 'youtube_ownership', 'yt_api_log') == 0 def test_get_stripspecialchars_func_name(sources_mock): """Test get_stripspecialchars_func_name method.""" assert sources_mock.get_stripspecialchars_func_name( 'ows_rds', 'youtube_ownership', 'yt_api_log') == 'stripSpecialChars' assert sources_mock.get_stripspecialchars_func_name( 'ows_rds', 'sales_goals', 'sales_goal') == 'stripSpecialCharsLong' assert sources_mock.get_stripspecialchars_func_name( 'research_collections', 'ppb_collections', 'track_delivery_history_7') == '' def test_is_schema_supported(sources_mock): """Test is_schema_supported method.""" assert sources_mock.is_schema_supported('ows_rds', 'youtube_ownership') assert not sources_mock.is_schema_supported( 'ows_rds', 'nonexistent_schema') def test_is_table_supported(sources_mock): """Test is_table_supported method.""" assert sources_mock.is_table_supported( 'ows_rds', 'youtube_ownership', 'yt_api_log') assert not sources_mock.is_table_supported( 'ows_rds', 'nonexistent_schema', 'nonexistent_table') def test_is_table_transient(sources_mock): """Test is_table_transient method.""" # if schema is not described in sql2sf_sources.yml with pytest.raises(KeyError): assert sources_mock.is_table_transient( 'redshift04', 'nonexistent_schema', 'dim_artist') # if table is not described in sql2sf_sources.yml within an existing schema with pytest.raises(KeyError): assert not sources_mock.is_table_transient( 'redshift04', 'production', 'nonexistent_table') # all dim_ tables are transient assert sources_mock.is_table_transient( 'redshift04', 'production', 'dim_artist') == 'TRANSIENT' # all fact_tables are permanent assert not sources_mock.is_table_transient( 'redshift04', 'production', 'fact_analytics') def test_is_schema_validation_needed(sources_mock): """Test is_table_supported method.""" assert sources_mock.is_schema_validation_needed( 'ows_rds', 'youtube_ownership', 'yt_api_log') assert not sources_mock.is_schema_validation_needed( 'ows_rds', 'sales_goals', 'sales_goal')