"""Unit tests for common functions regarding file open, read, write.""" import codecs import tempfile from unittest.mock import patch, call import pytest from feed_ingestion.util import file_operations def test_remove_bom(): """Test remove_bom.""" # We need delete=False when we need to close file explicitly, # so the content will be available for removing bom with tempfile.NamedTemporaryFile(delete=False) as read_file, \ tempfile.NamedTemporaryFile() as write_file: read_file.write( b'\xef\xbb\xbf30 for 30: Into the Wind 889176830402') read_file.close() file_source_path = read_file.name destination_clean_s3_path = write_file.name response = file_operations.remove_bom( file_source_path, destination_clean_s3_path) assert response is None expected = open(write_file.name) expected_content = expected.read().encode('utf8') assert expected_content == \ b'30 for 30: Into the Wind 889176830402' assert expected_content.startswith(codecs.BOM_UTF8) is False def test_remove_footer(): """Test remove_footer.""" read_file = tempfile.NamedTemporaryFile(delete=False) read_file.write(b'30 for 30: Into the Wind 889176830402') read_file.close() write_file = tempfile.NamedTemporaryFile(delete=False) write_file.close() destination_tmp_path = read_file.name number_of_line = 1 response = file_operations.remove_footer( destination_tmp_path, number_of_line) assert response is None expected = open(write_file.name) expected_content = expected.read().encode('utf8') assert expected_content == b'' def test_remove_only_specific_footer(): """Test remove_footer when footer_with_text don't match footer in file.""" read_file = tempfile.NamedTemporaryFile(delete=False) read_file.write(b'30 for 30: Into the Wind 889176830402') destination_tmp_path = read_file.name read_file.close() response = file_operations.remove_footer( destination_tmp_path, 1, 'no match') assert response is None expected = open(destination_tmp_path) expected_content = expected.read().encode('utf8') assert expected_content == b'30 for 30: Into the Wind 889176830402' def test_remove_when_footer_match(): """Test remove_footer when footer_with_text matches footer in file.""" read_file = tempfile.NamedTemporaryFile(delete=False) read_file.write(b'30 for 30: Into the Wind\nGrand Total: 100') destination_tmp_path = read_file.name read_file.close() response = file_operations.remove_footer( destination_tmp_path, 1, 'Grand Total') assert response is None expected = open(destination_tmp_path) expected_content = expected.read().encode('utf8') assert expected_content == b'30 for 30: Into the Wind\n' def test_open_with_encodings(): """Test open_with_encodings.""" with tempfile.NamedTemporaryFile(mode='w+', encoding='iso8859-1') as tf: test_string = u'\xf1 foobar' tf.write(test_string) tf.seek(0) with file_operations.open_with_encodings( tf.name, encodings=['iso8859-1']) as fh: assert fh.read() == test_string # correct encoding guess with file_operations.open_with_encodings( tf.name, encodings=['iso8859-1']) as fh: assert fh.read() == test_string # even though this encoding is different, the code points are # compatible with file_operations.open_with_encodings( tf.name, encodings=['iso8859-15']) as fh: assert fh.read() == test_string # defaults to trying utf8, the \xf1 character should be incompatible with pytest.raises(UnicodeError): with file_operations.open_with_encodings(tf.name) as fh: pass @patch.object(file_operations.subprocess, 'run') @patch.object(file_operations, 'THIS_DIR', '/path/to/this/dir') def test_split_file(run_mock): result = file_operations.split_file( source_file='tests/test_file_operations.gz', destination_path='s3://dev-cucumbers/tests/util/split_files', chunk_size='1000m', compression_level=5, cut_header_lines=1, archiver='gzip' ) assert run_mock.call_args_list == [ call(['bash', '/path/to/this/dir/split_file.sh', '-s', 'tests/test_file_operations.gz', '-t', 's3://dev-cucumbers/tests/util/split_files', '-z', '1000m', '-c', '5', '-d', '1', '-a', 'gzip'], stdout=-1, stderr=-1, text=True ) ] assert result == { 'result': False, 'return_code': run_mock.return_value.returncode, 'stderr': run_mock.return_value.stderr, 'stdout': run_mock.return_value.stdout, }