""" Tests for cloud storage adaptors. """ import os import sys import pytest import tempfile from pathlib import Path from unittest.mock import Mock, patch from skill_seekers.cli.storage import ( get_storage_adaptor, BaseStorageAdaptor, S3StorageAdaptor, GCSStorageAdaptor, AzureStorageAdaptor, StorageObject, ) # Check if cloud storage dependencies are available try: import boto3 # noqa: F401 BOTO3_AVAILABLE = True except ImportError: BOTO3_AVAILABLE = False try: from google.cloud import storage # noqa: F401 GCS_AVAILABLE = True except ImportError: GCS_AVAILABLE = False try: from azure.storage.blob import BlobServiceClient # noqa: F401 AZURE_AVAILABLE = True except ImportError: AZURE_AVAILABLE = False # ======================================== # Factory Tests # ======================================== def test_get_storage_adaptor_s3(): """Test S3 adaptor factory.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3'): adaptor = get_storage_adaptor('s3', bucket='test-bucket') assert isinstance(adaptor, S3StorageAdaptor) def test_get_storage_adaptor_gcs(): """Test GCS adaptor factory.""" if not GCS_AVAILABLE: pytest.skip("google-cloud-storage not installed") with patch('skill_seekers.cli.storage.gcs_storage.storage'): adaptor = get_storage_adaptor('gcs', bucket='test-bucket') assert isinstance(adaptor, GCSStorageAdaptor) def test_get_storage_adaptor_azure(): """Test Azure adaptor factory.""" if not AZURE_AVAILABLE: pytest.skip("azure-storage-blob not installed") with patch('skill_seekers.cli.storage.azure_storage.BlobServiceClient'): adaptor = get_storage_adaptor( 'azure', container='test-container', connection_string='DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key' ) assert isinstance(adaptor, AzureStorageAdaptor) def test_get_storage_adaptor_invalid_provider(): """Test invalid provider raises error.""" with pytest.raises(ValueError, match="Unsupported storage provider"): get_storage_adaptor('invalid', bucket='test') # ======================================== # S3 Storage Tests # ======================================== def test_s3_upload_file(): """Test S3 file upload.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3') as mock_boto3: # Setup mocks mock_client = Mock() mock_boto3.client.return_value = mock_client mock_boto3.resource.return_value = Mock() adaptor = S3StorageAdaptor(bucket='test-bucket') # Create temporary file with tempfile.NamedTemporaryFile(delete=False) as tmp_file: tmp_file.write(b'test content') tmp_path = tmp_file.name try: # Test upload result = adaptor.upload_file(tmp_path, 'test.txt') assert result == 's3://test-bucket/test.txt' mock_client.upload_file.assert_called_once() finally: Path(tmp_path).unlink() def test_s3_download_file(): """Test S3 file download.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3') as mock_boto3: # Setup mocks mock_client = Mock() mock_boto3.client.return_value = mock_client mock_boto3.resource.return_value = Mock() adaptor = S3StorageAdaptor(bucket='test-bucket') with tempfile.TemporaryDirectory() as tmp_dir: local_path = os.path.join(tmp_dir, 'downloaded.txt') # Test download adaptor.download_file('test.txt', local_path) mock_client.download_file.assert_called_once_with( 'test-bucket', 'test.txt', local_path ) def test_s3_list_files(): """Test S3 file listing.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3') as mock_boto3: # Setup mocks mock_client = Mock() mock_paginator = Mock() mock_page_iterator = [ { 'Contents': [ { 'Key': 'file1.txt', 'Size': 100, 'LastModified': Mock(isoformat=lambda: '2024-01-01T00:00:00'), 'ETag': '"abc123"' } ] } ] mock_paginator.paginate.return_value = mock_page_iterator mock_client.get_paginator.return_value = mock_paginator mock_boto3.client.return_value = mock_client mock_boto3.resource.return_value = Mock() adaptor = S3StorageAdaptor(bucket='test-bucket') # Test list files = adaptor.list_files('prefix/') assert len(files) == 1 assert files[0].key == 'file1.txt' assert files[0].size == 100 assert files[0].etag == 'abc123' def test_s3_file_exists(): """Test S3 file existence check.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3') as mock_boto3: # Setup mocks mock_client = Mock() mock_client.head_object.return_value = {} mock_boto3.client.return_value = mock_client mock_boto3.resource.return_value = Mock() adaptor = S3StorageAdaptor(bucket='test-bucket') # Test exists assert adaptor.file_exists('test.txt') is True def test_s3_get_file_url(): """Test S3 presigned URL generation.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3') as mock_boto3: # Setup mocks mock_client = Mock() mock_client.generate_presigned_url.return_value = 'https://s3.amazonaws.com/signed-url' mock_boto3.client.return_value = mock_client mock_boto3.resource.return_value = Mock() adaptor = S3StorageAdaptor(bucket='test-bucket') # Test URL generation url = adaptor.get_file_url('test.txt', expires_in=7200) assert url == 'https://s3.amazonaws.com/signed-url' mock_client.generate_presigned_url.assert_called_once() # ======================================== # GCS Storage Tests # ======================================== def test_gcs_upload_file(): """Test GCS file upload.""" if not GCS_AVAILABLE: pytest.skip("google-cloud-storage not installed") with patch('skill_seekers.cli.storage.gcs_storage.storage') as mock_storage: # Setup mocks mock_client = Mock() mock_bucket = Mock() mock_blob = Mock() mock_client.bucket.return_value = mock_bucket mock_bucket.blob.return_value = mock_blob mock_storage.Client.return_value = mock_client adaptor = GCSStorageAdaptor(bucket='test-bucket') # Create temporary file with tempfile.NamedTemporaryFile(delete=False) as tmp_file: tmp_file.write(b'test content') tmp_path = tmp_file.name try: # Test upload result = adaptor.upload_file(tmp_path, 'test.txt') assert result == 'gs://test-bucket/test.txt' mock_blob.upload_from_filename.assert_called_once() finally: Path(tmp_path).unlink() def test_gcs_download_file(): """Test GCS file download.""" if not GCS_AVAILABLE: pytest.skip("google-cloud-storage not installed") with patch('skill_seekers.cli.storage.gcs_storage.storage') as mock_storage: # Setup mocks mock_client = Mock() mock_bucket = Mock() mock_blob = Mock() mock_client.bucket.return_value = mock_bucket mock_bucket.blob.return_value = mock_blob mock_storage.Client.return_value = mock_client adaptor = GCSStorageAdaptor(bucket='test-bucket') with tempfile.TemporaryDirectory() as tmp_dir: local_path = os.path.join(tmp_dir, 'downloaded.txt') # Test download adaptor.download_file('test.txt', local_path) mock_blob.download_to_filename.assert_called_once() def test_gcs_list_files(): """Test GCS file listing.""" if not GCS_AVAILABLE: pytest.skip("google-cloud-storage not installed") with patch('skill_seekers.cli.storage.gcs_storage.storage') as mock_storage: # Setup mocks mock_client = Mock() mock_blob = Mock() mock_blob.name = 'file1.txt' mock_blob.size = 100 mock_blob.updated = Mock(isoformat=lambda: '2024-01-01T00:00:00') mock_blob.etag = 'abc123' mock_blob.metadata = {} mock_client.list_blobs.return_value = [mock_blob] mock_storage.Client.return_value = mock_client mock_client.bucket.return_value = Mock() adaptor = GCSStorageAdaptor(bucket='test-bucket') # Test list files = adaptor.list_files('prefix/') assert len(files) == 1 assert files[0].key == 'file1.txt' assert files[0].size == 100 # ======================================== # Azure Storage Tests # ======================================== def test_azure_upload_file(): """Test Azure file upload.""" if not AZURE_AVAILABLE: pytest.skip("azure-storage-blob not installed") with patch('skill_seekers.cli.storage.azure_storage.BlobServiceClient') as mock_blob_service: # Setup mocks mock_service_client = Mock() mock_container_client = Mock() mock_blob_client = Mock() mock_service_client.get_container_client.return_value = mock_container_client mock_container_client.get_blob_client.return_value = mock_blob_client mock_blob_service.from_connection_string.return_value = mock_service_client connection_string = 'DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key' adaptor = AzureStorageAdaptor(container='test-container', connection_string=connection_string) # Create temporary file with tempfile.NamedTemporaryFile(delete=False) as tmp_file: tmp_file.write(b'test content') tmp_path = tmp_file.name try: # Test upload result = adaptor.upload_file(tmp_path, 'test.txt') assert 'test.blob.core.windows.net' in result mock_blob_client.upload_blob.assert_called_once() finally: Path(tmp_path).unlink() def test_azure_download_file(): """Test Azure file download.""" if not AZURE_AVAILABLE: pytest.skip("azure-storage-blob not installed") with patch('skill_seekers.cli.storage.azure_storage.BlobServiceClient') as mock_blob_service: # Setup mocks mock_service_client = Mock() mock_container_client = Mock() mock_blob_client = Mock() mock_download_stream = Mock() mock_download_stream.readall.return_value = b'test content' mock_service_client.get_container_client.return_value = mock_container_client mock_container_client.get_blob_client.return_value = mock_blob_client mock_blob_client.download_blob.return_value = mock_download_stream mock_blob_service.from_connection_string.return_value = mock_service_client connection_string = 'DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key' adaptor = AzureStorageAdaptor(container='test-container', connection_string=connection_string) with tempfile.TemporaryDirectory() as tmp_dir: local_path = os.path.join(tmp_dir, 'downloaded.txt') # Test download adaptor.download_file('test.txt', local_path) assert Path(local_path).exists() assert Path(local_path).read_bytes() == b'test content' def test_azure_list_files(): """Test Azure file listing.""" if not AZURE_AVAILABLE: pytest.skip("azure-storage-blob not installed") with patch('skill_seekers.cli.storage.azure_storage.BlobServiceClient') as mock_blob_service: # Setup mocks mock_service_client = Mock() mock_container_client = Mock() mock_blob = Mock() mock_blob.name = 'file1.txt' mock_blob.size = 100 mock_blob.last_modified = Mock(isoformat=lambda: '2024-01-01T00:00:00') mock_blob.etag = 'abc123' mock_blob.metadata = {} mock_container_client.list_blobs.return_value = [mock_blob] mock_service_client.get_container_client.return_value = mock_container_client mock_blob_service.from_connection_string.return_value = mock_service_client connection_string = 'DefaultEndpointsProtocol=https;AccountName=test;AccountKey=key' adaptor = AzureStorageAdaptor(container='test-container', connection_string=connection_string) # Test list files = adaptor.list_files('prefix/') assert len(files) == 1 assert files[0].key == 'file1.txt' assert files[0].size == 100 # ======================================== # Base Adaptor Tests # ======================================== def test_storage_object(): """Test StorageObject dataclass.""" obj = StorageObject( key='test.txt', size=100, last_modified='2024-01-01T00:00:00', etag='abc123', metadata={'key': 'value'} ) assert obj.key == 'test.txt' assert obj.size == 100 assert obj.metadata == {'key': 'value'} def test_base_adaptor_abstract(): """Test that BaseStorageAdaptor cannot be instantiated.""" with pytest.raises(TypeError): BaseStorageAdaptor(bucket='test') # ======================================== # Integration-style Tests # ======================================== def test_upload_directory(): """Test directory upload.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3') as mock_boto3: # Setup mocks mock_client = Mock() mock_boto3.client.return_value = mock_client mock_boto3.resource.return_value = Mock() adaptor = S3StorageAdaptor(bucket='test-bucket') # Create temporary directory with files with tempfile.TemporaryDirectory() as tmp_dir: (Path(tmp_dir) / 'file1.txt').write_text('content1') (Path(tmp_dir) / 'file2.txt').write_text('content2') (Path(tmp_dir) / 'subdir').mkdir() (Path(tmp_dir) / 'subdir' / 'file3.txt').write_text('content3') # Test upload directory uploaded_files = adaptor.upload_directory(tmp_dir, 'skills/') assert len(uploaded_files) == 3 assert mock_client.upload_file.call_count == 3 def test_download_directory(): """Test directory download.""" if not BOTO3_AVAILABLE: pytest.skip("boto3 not installed") with patch('skill_seekers.cli.storage.s3_storage.boto3') as mock_boto3: # Setup mocks mock_client = Mock() mock_paginator = Mock() mock_page_iterator = [ { 'Contents': [ { 'Key': 'skills/file1.txt', 'Size': 100, 'LastModified': Mock(isoformat=lambda: '2024-01-01T00:00:00'), 'ETag': '"abc"' }, { 'Key': 'skills/file2.txt', 'Size': 200, 'LastModified': Mock(isoformat=lambda: '2024-01-01T00:00:00'), 'ETag': '"def"' } ] } ] mock_paginator.paginate.return_value = mock_page_iterator mock_client.get_paginator.return_value = mock_paginator mock_boto3.client.return_value = mock_client mock_boto3.resource.return_value = Mock() adaptor = S3StorageAdaptor(bucket='test-bucket') with tempfile.TemporaryDirectory() as tmp_dir: # Test download directory downloaded_files = adaptor.download_directory('skills/', tmp_dir) assert len(downloaded_files) == 2 assert mock_client.download_file.call_count == 2 def test_missing_dependencies(): """Test graceful handling of missing dependencies.""" # This test verifies that the modules can be imported even without optional deps # The actual import checks are done at class initialization time assert S3StorageAdaptor is not None assert GCSStorageAdaptor is not None assert AzureStorageAdaptor is not None