"""Property-based tests for Scanner behavior. **Validates: Requirements 1.3, 1.4, 1.5, 1.7** Property 2: Authentication error descriptiveness For any provider type and any authentication failure reason, the error returned by the Scanner SHALL contain both the provider name string and the failure reason string. Property 3: Graceful degradation on unsupported resource types For any scan request containing a mix of supported and unsupported resource types, the Scanner SHALL produce warnings for each unsupported type AND return a complete inventory for all supported types. Property 4: Progress reporting frequency The Scanner SHALL report progress at least once per resource type completion. Property 5: Partial inventory preservation on failure If the Provider API connection is lost during an active scan, the Scanner SHALL return a partial resource inventory. """ from typing import Callable import pytest from hypothesis import given, settings from hypothesis import strategies as st from iac_reverse.models import ( CpuArchitecture, DiscoveredResource, PlatformCategory, PROVIDER_SUPPORTED_RESOURCE_TYPES, ProviderType, ScanProfile, ScanProgress, ScanResult, ) from iac_reverse.plugin_base import ProviderPlugin from iac_reverse.scanner.scanner import ( AuthenticationError, ConnectionLostError, Scanner, ) # --------------------------------------------------------------------------- # Hypothesis Strategies # --------------------------------------------------------------------------- provider_type_strategy = st.sampled_from(list(ProviderType)) non_empty_string_strategy = st.text( min_size=1, max_size=100, alphabet=st.characters(whitelist_categories=("L", "N", "P", "S")), ).filter(lambda s: s.strip()) non_empty_credentials_strategy = st.dictionaries( keys=st.text(min_size=1, max_size=20, alphabet=st.characters(whitelist_categories=("L", "N"))), values=st.text(min_size=1, max_size=50), min_size=1, max_size=5, ) unsupported_resource_type_strategy = st.text( min_size=5, max_size=30, alphabet=st.characters(whitelist_categories=("L",)), ).filter( lambda t: all(t not in types for types in PROVIDER_SUPPORTED_RESOURCE_TYPES.values()) ) # --------------------------------------------------------------------------- # Mock Plugin Implementations # --------------------------------------------------------------------------- class FailingAuthPlugin(ProviderPlugin): """A plugin that always fails authentication with a given reason.""" def __init__(self, failure_reason: str): self.failure_reason = failure_reason def authenticate(self, credentials: dict[str, str]) -> None: raise RuntimeError(self.failure_reason) def get_platform_category(self) -> PlatformCategory: return PlatformCategory.CONTAINER_ORCHESTRATION def list_endpoints(self) -> list[str]: return ["http://localhost:8080"] def list_supported_resource_types(self) -> list[str]: return ["mock_resource"] def detect_architecture(self, endpoint: str) -> CpuArchitecture: return CpuArchitecture.AMD64 def discover_resources( self, endpoints: list[str], resource_types: list[str], progress_callback: Callable[[ScanProgress], None], ) -> ScanResult: return ScanResult( resources=[], warnings=[], errors=[], scan_timestamp="", profile_hash="", ) class GracefulDegradationPlugin(ProviderPlugin): """A plugin that supports specific resource types and discovers resources for them.""" def __init__(self, supported_types: list[str]): self._supported_types = supported_types def authenticate(self, credentials: dict[str, str]) -> None: pass # Always succeeds def get_platform_category(self) -> PlatformCategory: return PlatformCategory.CONTAINER_ORCHESTRATION def list_endpoints(self) -> list[str]: return ["http://localhost:8080"] def list_supported_resource_types(self) -> list[str]: return self._supported_types def detect_architecture(self, endpoint: str) -> CpuArchitecture: return CpuArchitecture.AMD64 def discover_resources( self, endpoints: list[str], resource_types: list[str], progress_callback: Callable[[ScanProgress], None], ) -> ScanResult: # Create one resource per supported resource type requested resources = [] for i, rt in enumerate(resource_types): resources.append( DiscoveredResource( resource_type=rt, unique_id=f"id-{rt}-{i}", name=f"resource-{rt}-{i}", provider=ProviderType.KUBERNETES, platform_category=PlatformCategory.CONTAINER_ORCHESTRATION, architecture=CpuArchitecture.AMD64, endpoint="http://localhost:8080", attributes={"key": "value"}, ) ) progress_callback(ScanProgress( current_resource_type=rt, resources_discovered=i + 1, resource_types_completed=i + 1, total_resource_types=len(resource_types), )) return ScanResult( resources=resources, warnings=[], errors=[], scan_timestamp="", profile_hash="", ) class ProgressTrackingPlugin(ProviderPlugin): """A plugin that reports progress per resource type.""" def __init__(self, supported_types: list[str]): self._supported_types = supported_types def authenticate(self, credentials: dict[str, str]) -> None: pass def get_platform_category(self) -> PlatformCategory: return PlatformCategory.CONTAINER_ORCHESTRATION def list_endpoints(self) -> list[str]: return ["http://localhost:8080"] def list_supported_resource_types(self) -> list[str]: return self._supported_types def detect_architecture(self, endpoint: str) -> CpuArchitecture: return CpuArchitecture.AMD64 def discover_resources( self, endpoints: list[str], resource_types: list[str], progress_callback: Callable[[ScanProgress], None], ) -> ScanResult: resources = [] for i, rt in enumerate(resource_types): resource = DiscoveredResource( resource_type=rt, unique_id=f"id-{rt}-{i}", name=f"resource-{rt}-{i}", provider=ProviderType.KUBERNETES, platform_category=PlatformCategory.CONTAINER_ORCHESTRATION, architecture=CpuArchitecture.AMD64, endpoint="http://localhost:8080", attributes={}, ) resources.append(resource) progress_callback(ScanProgress( current_resource_type=rt, resources_discovered=i + 1, resource_types_completed=i + 1, total_resource_types=len(resource_types), )) return ScanResult( resources=resources, warnings=[], errors=[], scan_timestamp="", profile_hash="", ) class ConnectionLossPlugin(ProviderPlugin): """A plugin that loses connection after discovering some resources.""" def __init__(self, supported_types: list[str], fail_after: int): self._supported_types = supported_types self._fail_after = fail_after def authenticate(self, credentials: dict[str, str]) -> None: pass def get_platform_category(self) -> PlatformCategory: return PlatformCategory.CONTAINER_ORCHESTRATION def list_endpoints(self) -> list[str]: return ["http://localhost:8080"] def list_supported_resource_types(self) -> list[str]: return self._supported_types def detect_architecture(self, endpoint: str) -> CpuArchitecture: return CpuArchitecture.AMD64 def discover_resources( self, endpoints: list[str], resource_types: list[str], progress_callback: Callable[[ScanProgress], None], ) -> ScanResult: # Simulate connection loss by raising ConnectionError raise ConnectionError( f"Connection lost after discovering {self._fail_after} resources" ) # --------------------------------------------------------------------------- # Property Tests # --------------------------------------------------------------------------- class TestAuthenticationErrorDescriptiveness: """Property 2: Authentication error descriptiveness. For any provider type and any authentication failure reason, the error returned by the Scanner SHALL contain both the provider name string and the failure reason string. **Validates: Requirements 1.3** """ @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, failure_reason=non_empty_string_strategy, ) @settings(max_examples=100) def test_auth_error_contains_provider_name_and_reason( self, provider, credentials, failure_reason ): """AuthenticationError must contain both provider name and failure reason.""" plugin = FailingAuthPlugin(failure_reason=failure_reason) profile = ScanProfile( provider=provider, credentials=credentials, ) scanner = Scanner(profile=profile, plugin=plugin) with pytest.raises(AuthenticationError) as exc_info: scanner.scan() error = exc_info.value # The error must contain the provider name assert provider.value in str(error), ( f"Expected provider name '{provider.value}' in error message, " f"got: '{str(error)}'" ) # The error must contain the failure reason assert failure_reason in str(error), ( f"Expected failure reason '{failure_reason}' in error message, " f"got: '{str(error)}'" ) @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, failure_reason=non_empty_string_strategy, ) @settings(max_examples=100) def test_auth_error_attributes_match(self, provider, credentials, failure_reason): """AuthenticationError attributes must store provider_name and reason.""" plugin = FailingAuthPlugin(failure_reason=failure_reason) profile = ScanProfile( provider=provider, credentials=credentials, ) scanner = Scanner(profile=profile, plugin=plugin) with pytest.raises(AuthenticationError) as exc_info: scanner.scan() error = exc_info.value assert error.provider_name == provider.value assert error.reason == failure_reason class TestGracefulDegradationOnUnsupportedTypes: """Property 3: Graceful degradation on unsupported resource types. For any scan request containing a mix of supported and unsupported resource types, the Scanner SHALL produce warnings for each unsupported type AND return a complete inventory for all supported types. **Validates: Requirements 1.4** """ @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, unsupported_types=st.lists(unsupported_resource_type_strategy, min_size=1, max_size=5), ) @settings(max_examples=100) def test_unsupported_types_produce_warnings( self, provider, credentials, unsupported_types ): """Each unsupported resource type must produce a warning.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = GracefulDegradationPlugin(supported_types=supported) # Mix supported and unsupported types mixed_filters = list(supported[:2]) + unsupported_types profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=mixed_filters, ) scanner = Scanner(profile=profile, plugin=plugin) result = scanner.scan() # There must be a warning for each unsupported type for unsupported in unsupported_types: assert any(unsupported in w for w in result.warnings), ( f"Expected warning for unsupported type '{unsupported}', " f"got warnings: {result.warnings}" ) @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, unsupported_types=st.lists(unsupported_resource_type_strategy, min_size=1, max_size=5), ) @settings(max_examples=100) def test_supported_types_still_discovered( self, provider, credentials, unsupported_types ): """Supported types must still be fully discovered despite unsupported types.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = GracefulDegradationPlugin(supported_types=supported) # Use at least one supported type plus unsupported types supported_subset = supported[:2] mixed_filters = supported_subset + unsupported_types profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=mixed_filters, ) scanner = Scanner(profile=profile, plugin=plugin) result = scanner.scan() # All supported types in the filter should have resources discovered discovered_types = {r.resource_type for r in result.resources} for st_type in supported_subset: assert st_type in discovered_types, ( f"Expected supported type '{st_type}' to be discovered, " f"but only found: {discovered_types}" ) @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, unsupported_types=st.lists(unsupported_resource_type_strategy, min_size=1, max_size=5), ) @settings(max_examples=100) def test_warning_count_matches_unsupported_count( self, provider, credentials, unsupported_types ): """Number of warnings must be at least the number of unsupported types.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = GracefulDegradationPlugin(supported_types=supported) # Only unsupported types in filter profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=unsupported_types, ) scanner = Scanner(profile=profile, plugin=plugin) result = scanner.scan() # Deduplicate unsupported types for comparison unique_unsupported = set(unsupported_types) assert len(result.warnings) >= len(unique_unsupported), ( f"Expected at least {len(unique_unsupported)} warnings, " f"got {len(result.warnings)}: {result.warnings}" ) class TestProgressReportingFrequency: """Property 4: Progress reporting frequency. For any scan across N resource types, the progress callback SHALL be invoked at least N times, once per resource type completion. **Validates: Requirements 1.5** """ @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, ) @settings(max_examples=100) def test_progress_reported_at_least_once_per_resource_type( self, provider, credentials ): """Progress callback must be invoked at least once per resource type.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = ProgressTrackingPlugin(supported_types=supported) profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=None, # Scan all supported types ) progress_reports: list[ScanProgress] = [] def track_progress(progress: ScanProgress) -> None: progress_reports.append(progress) scanner = Scanner(profile=profile, plugin=plugin) scanner.scan(progress_callback=track_progress) # Must have at least N progress reports for N resource types assert len(progress_reports) >= len(supported), ( f"Expected at least {len(supported)} progress reports, " f"got {len(progress_reports)}" ) @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, ) @settings(max_examples=100) def test_progress_reports_cover_all_resource_types( self, provider, credentials ): """Progress reports must cover every resource type being scanned.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = ProgressTrackingPlugin(supported_types=supported) profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=None, ) progress_reports: list[ScanProgress] = [] def track_progress(progress: ScanProgress) -> None: progress_reports.append(progress) scanner = Scanner(profile=profile, plugin=plugin) scanner.scan(progress_callback=track_progress) # Every resource type should appear in at least one progress report reported_types = {p.current_resource_type for p in progress_reports} for rt in supported: assert rt in reported_types, ( f"Expected resource type '{rt}' in progress reports, " f"but only found: {reported_types}" ) class TestPartialInventoryPreservationOnFailure: """Property 5: Partial inventory preservation on failure. If the Provider API connection is lost during an active scan, the Scanner SHALL return a partial resource inventory. **Validates: Requirements 1.7** """ @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, fail_after=st.integers(min_value=0, max_value=10), ) @settings(max_examples=100) def test_connection_loss_raises_with_partial_result( self, provider, credentials, fail_after ): """Connection loss must raise ConnectionLostError with partial result.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = ConnectionLossPlugin( supported_types=supported, fail_after=fail_after, ) profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=None, ) scanner = Scanner(profile=profile, plugin=plugin) with pytest.raises(ConnectionLostError) as exc_info: scanner.scan() error = exc_info.value # Must have a partial_result attribute assert hasattr(error, "partial_result") partial = error.partial_result assert isinstance(partial, ScanResult) @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, fail_after=st.integers(min_value=0, max_value=10), ) @settings(max_examples=100) def test_partial_result_is_marked_as_partial( self, provider, credentials, fail_after ): """Partial result from connection loss must have is_partial=True.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = ConnectionLossPlugin( supported_types=supported, fail_after=fail_after, ) profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=None, ) scanner = Scanner(profile=profile, plugin=plugin) with pytest.raises(ConnectionLostError) as exc_info: scanner.scan() partial = exc_info.value.partial_result assert partial.is_partial is True, ( "Partial result from connection loss must have is_partial=True" ) @given( provider=provider_type_strategy, credentials=non_empty_credentials_strategy, fail_after=st.integers(min_value=0, max_value=10), ) @settings(max_examples=100) def test_partial_result_contains_error_info( self, provider, credentials, fail_after ): """Partial result must contain error information about the failure.""" supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider] plugin = ConnectionLossPlugin( supported_types=supported, fail_after=fail_after, ) profile = ScanProfile( provider=provider, credentials=credentials, resource_type_filters=None, ) scanner = Scanner(profile=profile, plugin=plugin) with pytest.raises(ConnectionLostError) as exc_info: scanner.scan() partial = exc_info.value.partial_result # Must have at least one error or warning indicating the failure assert len(partial.errors) > 0 or len(partial.warnings) > 0, ( "Partial result must contain error/warning info about the connection loss" )