"""Tests for the s32sf flow's helpers.""" import random import pytest from snowflake_etl.flows.s32sf import helpers @pytest.fixture def random_list_of_correct_file_format_options(): """Fixture returning a generator of correct file_format options lists.""" valid_file_format_options = [ 'TYPE="CSV"', 'TYPE="JSON"', 'TYPE="AVRO"', 'TYPE="XML"', 'COMPRESSION="AUTO"', 'COMPRESSION="GZIP"', 'COMPRESSION="BZ2"', 'COMPRESSION="DEFLATE"', 'COMPRESSION="RAW_DEFLATE"', 'COMPRESSION="NONE"', 'ESCAPE="\\134"', 'ESCAPE_UNENCLOSED_FIELD="\\134"', 'RECORD_DELIMITER="\\n"', 'RECORD_DELIMITER="\\r"', 'RECORD_DELIMITER="NONE"', 'FIELD_DELIMITER=","', 'FIELD_DELIMITER=";"', 'FIELD_DELIMITER="|"', 'FIELD_DELIMITER="\\t"', 'FIELD_DELIMITER=" "', 'FIELD_DELIMITER="NONE"', 'NULL_IF=("__NULL__")', 'NULL_IF=("NULL")', "NULL_IF=()", # noqa 'FIELD_OPTIONALLY_ENCLOSED_BY="\'"', 'FIELD_OPTIONALLY_ENCLOSED_BY="\""', 'FIELD_OPTIONALLY_ENCLOSED_BY="NONE"', 'TRIM_SPACE=TRUE', 'TRIM_SPACE=FALSE', 'ERROR_ON_COLUMN_COUNT_MISMATCH=TRUE', 'ERROR_ON_COLUMN_COUNT_MISMATCH=FALSE', 'ENABLE_OCTAL=TRUE', 'ENABLE_OCTAL=FALSE', 'ALLOW_DUPLICATE=TRUE', 'ALLOW_DUPLICATE=FALSE', 'STRIP_OUTER_ARRAY=TRUE', 'STRIP_OUTER_ARRAY=FALSE', 'STRIP_NULL_VALUES=TRUE', 'STRIP_NULL_VALUES=FALSE', 'IGNORE_UTF8_ERRORS=TRUE', 'IGNORE_UTF8_ERRORS=FALSE', 'PRESERVE_SPACE=TRUE', 'PRESERVE_SPACE=FALSE', 'STRIP_OUTER_ELEMENT=TRUE', 'STRIP_OUTER_ELEMENT=FALSE', 'DISABLE_SNOWFLAKE_DATA=TRUE', 'DISABLE_SNOWFLAKE_DATA=FALSE', 'DISABLE_AUTO_CONVERT=TRUE', 'DISABLE_AUTO_CONVERT=FALSE' ] def rand_list(): random_list = [] added = [] random.shuffle(valid_file_format_options) for option in valid_file_format_options: if option.split('=')[0] not in added: random_list.append(option) added.append(option.split('=')[0]) yield random_list return rand_list def test_validate_file_format(random_list_of_correct_file_format_options): """Test successful validation of correct file_format.""" for i in range(5): helpers.validate_file_format( next(random_list_of_correct_file_format_options())) def test_validate_file_format_failure_due_to_wrong_option(): """Test failed validation of incorrect file_format.""" with pytest.raises(ValueError): helpers.validate_file_format( ["'TYPE='CSV'", "'ESCAPE='\134'", ";DROP ALL THE TABLES;"]) # noqa def test_validate_file_format_failure_due_to_wrong_option_value(): """Test failed validation of incorrect file_format.""" with pytest.raises(ValueError): helpers.validate_file_format( ["'TYPE='MP3;DROP ALL THE TABLES;'"]) def test_validate_file_format_failure_due_to_wrong_date_format(): """Test failed validation of incorrect file_format.""" with pytest.raises(ValueError): helpers.validate_file_format( ["'DATE_FORMAT='YYYYYY:_dd_MM'"]) def test_extract_db_and_schema(): """Test extraction of DB name and schema from SF config dict.""" db_value = 'test_db' schema_value = 'test_schema' cfg1 = dict(db=db_value, schema=schema_value, dummy='dummy') cfg2 = dict(test_db=db_value, test_schema=schema_value, dummy='dummy') res = helpers.extract_db_and_schema(cfg1) assert isinstance(res, tuple) db, schema = res assert db, db_value assert schema, schema_value db, schema = helpers.extract_db_and_schema(cfg2) assert not db assert not schema db, schema = helpers.extract_db_and_schema(cfg2, 'test_db', 'test_schema') assert db, db_value assert schema, schema_value db, schema = helpers.extract_db_and_schema(cfg1, 'test_db') assert not db assert schema, schema_value