import os from unittest import mock import pytest from cloud_storage_mocker import Mount from cloud_storage_mocker import patch as gcs_patch from cloud_storage_mocker._core import Blob from cloud_storage_mocker._core import Bucket from vector_utils.connections import connection_info from vector_utils.connections import gcs @pytest.fixture def mock_gcs_connection(tmp_path, gcs_conn_info): """Provides a patched GCSConnection pointing to a fake bucket.""" def exists(self): return self._get_local_path(readable=True).exists() Blob.exists = exists @property def _blob_size(self): path = self._get_local_path(readable=True) if not path.exists(): raise FileNotFoundError(f'No such file: {path}') return os.path.getsize(path) Blob.size = _blob_size def _get_blob(self, name: str): blob = self.blob(name) return blob if blob._get_local_path(readable=True).exists() else None Bucket.get_blob = _get_blob def _list_blobs(self, prefix: str = ''): mount = self._env.get_mount(self.name) base_dir = mount.directory / prefix if not base_dir.exists(): return [] results = [] for path in base_dir.rglob('*'): if path.is_file(): rel_name = str(path.relative_to(mount.directory)).replace('\\', '/') results.append(self.blob(rel_name)) return results Bucket.list_blobs = _list_blobs bucket_dir = tmp_path / 'test_bucket' (bucket_dir / 'a_dir').mkdir(parents=True) (bucket_dir / 'a_dir' / 'hello.txt').write_text('Hello World!') with gcs_patch( mounts=[Mount(gcs_conn_info['domain_name'], bucket_dir, readable=True, writable=True)], client_cls_names=['vector_utils.connections.gcs.Client', 'vector_utils.connections.gcs.Credentials'] ): conn_obj = connection_info.ConnectionInfo(gcs_conn_info) yield gcs.GCSConnection(conn_obj) def test_file_exists(mock_gcs_connection): """Test that file existence is correctly detected.""" assert mock_gcs_connection.file_exists('a_dir/hello.txt') assert not mock_gcs_connection.file_exists('a_dir/missing.txt') def test_file_size(mock_gcs_connection): """Test that file size is reported correctly.""" size = mock_gcs_connection.file_size('a_dir/hello.txt') assert size == 12 def test_file_size_exception_raised(mock_gcs_connection): """Test file size not found raises exception.""" with pytest.raises(FileNotFoundError) as excinfo: mock_gcs_connection.file_size('/a_dir/somefile111.txt') assert 'No such file' in str(excinfo) def test_mkdir(mock_gcs_connection): """Test that creating a new directory works.""" assert mock_gcs_connection.mkdir('new_dir/') is True assert mock_gcs_connection.mkdir('/a_dir') is None assert mock_gcs_connection.mkdir('/a_dir/') is None def test_scan_dir(mock_gcs_connection): """Test that scan_dir lists files while respecting exceptions.""" results = mock_gcs_connection.scan_dir('a_dir', exceptions=['ignore.txt']) assert 'a_dir/hello.txt' in results def test_upload_and_list(mock_gcs_connection, local_test_files): """Test uploading files and verifying they appear in scan_dir results.""" transfer_files_list = [ {'local': x, 'remote': os.path.basename(x)} for x in local_test_files ] mock_gcs_connection.transfer_files(transfer_files_list) results = mock_gcs_connection.scan_dir('') for x in local_test_files: assert os.path.basename(x) in [os.path.basename(r) for r in results] def test_download(mock_gcs_connection, tmpdir): """Test downloading a file from the bucket into the local filesystem.""" local_file_location = f'{tmpdir}/somefile.txt' transfer_files_list = [{ 'local': local_file_location, 'remote': 'a_dir/hello.txt' }] mock_gcs_connection.transfer_files(transfer_files_list, transfer_mode='download') assert tmpdir.join('somefile.txt').check() @mock.patch('vector_utils.connections.gcs.sleep') def test_connection_transfer_files_download_err(mock_sleep, mock_gcs_connection, tmpdir): """Test that download fails gracefully when remote file is missing.""" local_file_location = f'{tmpdir}/somefile.txt' transfer_files_list = [{ 'local': local_file_location, 'remote': '/a_dir/somefile_1.txt' }] with pytest.raises(Exception) as excinfo: mock_gcs_connection.transfer_files(transfer_files_list, transfer_mode='download') assert str(excinfo.value) == 'Unable to transfer files'