609 lines
21 KiB
Python
609 lines
21 KiB
Python
"""Property-based tests for Scanner behavior.
|
|
|
|
**Validates: Requirements 1.3, 1.4, 1.5, 1.7**
|
|
|
|
Property 2: Authentication error descriptiveness
|
|
For any provider type and any authentication failure reason, the error returned
|
|
by the Scanner SHALL contain both the provider name string and the failure reason string.
|
|
|
|
Property 3: Graceful degradation on unsupported resource types
|
|
For any scan request containing a mix of supported and unsupported resource types,
|
|
the Scanner SHALL produce warnings for each unsupported type AND return a complete
|
|
inventory for all supported types.
|
|
|
|
Property 4: Progress reporting frequency
|
|
The Scanner SHALL report progress at least once per resource type completion.
|
|
|
|
Property 5: Partial inventory preservation on failure
|
|
If the Provider API connection is lost during an active scan, the Scanner SHALL
|
|
return a partial resource inventory.
|
|
"""
|
|
|
|
from typing import Callable
|
|
|
|
import pytest
|
|
from hypothesis import given, settings
|
|
from hypothesis import strategies as st
|
|
|
|
from iac_reverse.models import (
|
|
CpuArchitecture,
|
|
DiscoveredResource,
|
|
PlatformCategory,
|
|
PROVIDER_SUPPORTED_RESOURCE_TYPES,
|
|
ProviderType,
|
|
ScanProfile,
|
|
ScanProgress,
|
|
ScanResult,
|
|
)
|
|
from iac_reverse.plugin_base import ProviderPlugin
|
|
from iac_reverse.scanner.scanner import (
|
|
AuthenticationError,
|
|
ConnectionLostError,
|
|
Scanner,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Hypothesis Strategies
|
|
# ---------------------------------------------------------------------------
|
|
|
|
provider_type_strategy = st.sampled_from(list(ProviderType))
|
|
|
|
non_empty_string_strategy = st.text(
|
|
min_size=1,
|
|
max_size=100,
|
|
alphabet=st.characters(whitelist_categories=("L", "N", "P", "S")),
|
|
).filter(lambda s: s.strip())
|
|
|
|
non_empty_credentials_strategy = st.dictionaries(
|
|
keys=st.text(min_size=1, max_size=20, alphabet=st.characters(whitelist_categories=("L", "N"))),
|
|
values=st.text(min_size=1, max_size=50),
|
|
min_size=1,
|
|
max_size=5,
|
|
)
|
|
|
|
unsupported_resource_type_strategy = st.text(
|
|
min_size=5,
|
|
max_size=30,
|
|
alphabet=st.characters(whitelist_categories=("L",)),
|
|
).filter(
|
|
lambda t: all(t not in types for types in PROVIDER_SUPPORTED_RESOURCE_TYPES.values())
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Mock Plugin Implementations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class FailingAuthPlugin(ProviderPlugin):
|
|
"""A plugin that always fails authentication with a given reason."""
|
|
|
|
def __init__(self, failure_reason: str):
|
|
self.failure_reason = failure_reason
|
|
|
|
def authenticate(self, credentials: dict[str, str]) -> None:
|
|
raise RuntimeError(self.failure_reason)
|
|
|
|
def get_platform_category(self) -> PlatformCategory:
|
|
return PlatformCategory.CONTAINER_ORCHESTRATION
|
|
|
|
def list_endpoints(self) -> list[str]:
|
|
return ["http://localhost:8080"]
|
|
|
|
def list_supported_resource_types(self) -> list[str]:
|
|
return ["mock_resource"]
|
|
|
|
def detect_architecture(self, endpoint: str) -> CpuArchitecture:
|
|
return CpuArchitecture.AMD64
|
|
|
|
def discover_resources(
|
|
self,
|
|
endpoints: list[str],
|
|
resource_types: list[str],
|
|
progress_callback: Callable[[ScanProgress], None],
|
|
) -> ScanResult:
|
|
return ScanResult(
|
|
resources=[], warnings=[], errors=[],
|
|
scan_timestamp="", profile_hash="",
|
|
)
|
|
|
|
|
|
class GracefulDegradationPlugin(ProviderPlugin):
|
|
"""A plugin that supports specific resource types and discovers resources for them."""
|
|
|
|
def __init__(self, supported_types: list[str]):
|
|
self._supported_types = supported_types
|
|
|
|
def authenticate(self, credentials: dict[str, str]) -> None:
|
|
pass # Always succeeds
|
|
|
|
def get_platform_category(self) -> PlatformCategory:
|
|
return PlatformCategory.CONTAINER_ORCHESTRATION
|
|
|
|
def list_endpoints(self) -> list[str]:
|
|
return ["http://localhost:8080"]
|
|
|
|
def list_supported_resource_types(self) -> list[str]:
|
|
return self._supported_types
|
|
|
|
def detect_architecture(self, endpoint: str) -> CpuArchitecture:
|
|
return CpuArchitecture.AMD64
|
|
|
|
def discover_resources(
|
|
self,
|
|
endpoints: list[str],
|
|
resource_types: list[str],
|
|
progress_callback: Callable[[ScanProgress], None],
|
|
) -> ScanResult:
|
|
# Create one resource per supported resource type requested
|
|
resources = []
|
|
for i, rt in enumerate(resource_types):
|
|
resources.append(
|
|
DiscoveredResource(
|
|
resource_type=rt,
|
|
unique_id=f"id-{rt}-{i}",
|
|
name=f"resource-{rt}-{i}",
|
|
provider=ProviderType.KUBERNETES,
|
|
platform_category=PlatformCategory.CONTAINER_ORCHESTRATION,
|
|
architecture=CpuArchitecture.AMD64,
|
|
endpoint="http://localhost:8080",
|
|
attributes={"key": "value"},
|
|
)
|
|
)
|
|
progress_callback(ScanProgress(
|
|
current_resource_type=rt,
|
|
resources_discovered=i + 1,
|
|
resource_types_completed=i + 1,
|
|
total_resource_types=len(resource_types),
|
|
))
|
|
return ScanResult(
|
|
resources=resources,
|
|
warnings=[],
|
|
errors=[],
|
|
scan_timestamp="",
|
|
profile_hash="",
|
|
)
|
|
|
|
|
|
class ProgressTrackingPlugin(ProviderPlugin):
|
|
"""A plugin that reports progress per resource type."""
|
|
|
|
def __init__(self, supported_types: list[str]):
|
|
self._supported_types = supported_types
|
|
|
|
def authenticate(self, credentials: dict[str, str]) -> None:
|
|
pass
|
|
|
|
def get_platform_category(self) -> PlatformCategory:
|
|
return PlatformCategory.CONTAINER_ORCHESTRATION
|
|
|
|
def list_endpoints(self) -> list[str]:
|
|
return ["http://localhost:8080"]
|
|
|
|
def list_supported_resource_types(self) -> list[str]:
|
|
return self._supported_types
|
|
|
|
def detect_architecture(self, endpoint: str) -> CpuArchitecture:
|
|
return CpuArchitecture.AMD64
|
|
|
|
def discover_resources(
|
|
self,
|
|
endpoints: list[str],
|
|
resource_types: list[str],
|
|
progress_callback: Callable[[ScanProgress], None],
|
|
) -> ScanResult:
|
|
resources = []
|
|
for i, rt in enumerate(resource_types):
|
|
resource = DiscoveredResource(
|
|
resource_type=rt,
|
|
unique_id=f"id-{rt}-{i}",
|
|
name=f"resource-{rt}-{i}",
|
|
provider=ProviderType.KUBERNETES,
|
|
platform_category=PlatformCategory.CONTAINER_ORCHESTRATION,
|
|
architecture=CpuArchitecture.AMD64,
|
|
endpoint="http://localhost:8080",
|
|
attributes={},
|
|
)
|
|
resources.append(resource)
|
|
progress_callback(ScanProgress(
|
|
current_resource_type=rt,
|
|
resources_discovered=i + 1,
|
|
resource_types_completed=i + 1,
|
|
total_resource_types=len(resource_types),
|
|
))
|
|
return ScanResult(
|
|
resources=resources,
|
|
warnings=[],
|
|
errors=[],
|
|
scan_timestamp="",
|
|
profile_hash="",
|
|
)
|
|
|
|
|
|
class ConnectionLossPlugin(ProviderPlugin):
|
|
"""A plugin that loses connection after discovering some resources."""
|
|
|
|
def __init__(self, supported_types: list[str], fail_after: int):
|
|
self._supported_types = supported_types
|
|
self._fail_after = fail_after
|
|
|
|
def authenticate(self, credentials: dict[str, str]) -> None:
|
|
pass
|
|
|
|
def get_platform_category(self) -> PlatformCategory:
|
|
return PlatformCategory.CONTAINER_ORCHESTRATION
|
|
|
|
def list_endpoints(self) -> list[str]:
|
|
return ["http://localhost:8080"]
|
|
|
|
def list_supported_resource_types(self) -> list[str]:
|
|
return self._supported_types
|
|
|
|
def detect_architecture(self, endpoint: str) -> CpuArchitecture:
|
|
return CpuArchitecture.AMD64
|
|
|
|
def discover_resources(
|
|
self,
|
|
endpoints: list[str],
|
|
resource_types: list[str],
|
|
progress_callback: Callable[[ScanProgress], None],
|
|
) -> ScanResult:
|
|
# Simulate connection loss by raising ConnectionError
|
|
raise ConnectionError(
|
|
f"Connection lost after discovering {self._fail_after} resources"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Property Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAuthenticationErrorDescriptiveness:
|
|
"""Property 2: Authentication error descriptiveness.
|
|
|
|
For any provider type and any authentication failure reason, the error
|
|
returned by the Scanner SHALL contain both the provider name string and
|
|
the failure reason string.
|
|
|
|
**Validates: Requirements 1.3**
|
|
"""
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
failure_reason=non_empty_string_strategy,
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_auth_error_contains_provider_name_and_reason(
|
|
self, provider, credentials, failure_reason
|
|
):
|
|
"""AuthenticationError must contain both provider name and failure reason."""
|
|
plugin = FailingAuthPlugin(failure_reason=failure_reason)
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
|
|
with pytest.raises(AuthenticationError) as exc_info:
|
|
scanner.scan()
|
|
|
|
error = exc_info.value
|
|
# The error must contain the provider name
|
|
assert provider.value in str(error), (
|
|
f"Expected provider name '{provider.value}' in error message, "
|
|
f"got: '{str(error)}'"
|
|
)
|
|
# The error must contain the failure reason
|
|
assert failure_reason in str(error), (
|
|
f"Expected failure reason '{failure_reason}' in error message, "
|
|
f"got: '{str(error)}'"
|
|
)
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
failure_reason=non_empty_string_strategy,
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_auth_error_attributes_match(self, provider, credentials, failure_reason):
|
|
"""AuthenticationError attributes must store provider_name and reason."""
|
|
plugin = FailingAuthPlugin(failure_reason=failure_reason)
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
|
|
with pytest.raises(AuthenticationError) as exc_info:
|
|
scanner.scan()
|
|
|
|
error = exc_info.value
|
|
assert error.provider_name == provider.value
|
|
assert error.reason == failure_reason
|
|
|
|
|
|
class TestGracefulDegradationOnUnsupportedTypes:
|
|
"""Property 3: Graceful degradation on unsupported resource types.
|
|
|
|
For any scan request containing a mix of supported and unsupported resource
|
|
types, the Scanner SHALL produce warnings for each unsupported type AND
|
|
return a complete inventory for all supported types.
|
|
|
|
**Validates: Requirements 1.4**
|
|
"""
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
unsupported_types=st.lists(unsupported_resource_type_strategy, min_size=1, max_size=5),
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_unsupported_types_produce_warnings(
|
|
self, provider, credentials, unsupported_types
|
|
):
|
|
"""Each unsupported resource type must produce a warning."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = GracefulDegradationPlugin(supported_types=supported)
|
|
|
|
# Mix supported and unsupported types
|
|
mixed_filters = list(supported[:2]) + unsupported_types
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=mixed_filters,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
result = scanner.scan()
|
|
|
|
# There must be a warning for each unsupported type
|
|
for unsupported in unsupported_types:
|
|
assert any(unsupported in w for w in result.warnings), (
|
|
f"Expected warning for unsupported type '{unsupported}', "
|
|
f"got warnings: {result.warnings}"
|
|
)
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
unsupported_types=st.lists(unsupported_resource_type_strategy, min_size=1, max_size=5),
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_supported_types_still_discovered(
|
|
self, provider, credentials, unsupported_types
|
|
):
|
|
"""Supported types must still be fully discovered despite unsupported types."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = GracefulDegradationPlugin(supported_types=supported)
|
|
|
|
# Use at least one supported type plus unsupported types
|
|
supported_subset = supported[:2]
|
|
mixed_filters = supported_subset + unsupported_types
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=mixed_filters,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
result = scanner.scan()
|
|
|
|
# All supported types in the filter should have resources discovered
|
|
discovered_types = {r.resource_type for r in result.resources}
|
|
for st_type in supported_subset:
|
|
assert st_type in discovered_types, (
|
|
f"Expected supported type '{st_type}' to be discovered, "
|
|
f"but only found: {discovered_types}"
|
|
)
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
unsupported_types=st.lists(unsupported_resource_type_strategy, min_size=1, max_size=5),
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_warning_count_matches_unsupported_count(
|
|
self, provider, credentials, unsupported_types
|
|
):
|
|
"""Number of warnings must be at least the number of unsupported types."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = GracefulDegradationPlugin(supported_types=supported)
|
|
|
|
# Only unsupported types in filter
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=unsupported_types,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
result = scanner.scan()
|
|
|
|
# Deduplicate unsupported types for comparison
|
|
unique_unsupported = set(unsupported_types)
|
|
assert len(result.warnings) >= len(unique_unsupported), (
|
|
f"Expected at least {len(unique_unsupported)} warnings, "
|
|
f"got {len(result.warnings)}: {result.warnings}"
|
|
)
|
|
|
|
|
|
class TestProgressReportingFrequency:
|
|
"""Property 4: Progress reporting frequency.
|
|
|
|
For any scan across N resource types, the progress callback SHALL be
|
|
invoked at least N times, once per resource type completion.
|
|
|
|
**Validates: Requirements 1.5**
|
|
"""
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_progress_reported_at_least_once_per_resource_type(
|
|
self, provider, credentials
|
|
):
|
|
"""Progress callback must be invoked at least once per resource type."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = ProgressTrackingPlugin(supported_types=supported)
|
|
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=None, # Scan all supported types
|
|
)
|
|
|
|
progress_reports: list[ScanProgress] = []
|
|
|
|
def track_progress(progress: ScanProgress) -> None:
|
|
progress_reports.append(progress)
|
|
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
scanner.scan(progress_callback=track_progress)
|
|
|
|
# Must have at least N progress reports for N resource types
|
|
assert len(progress_reports) >= len(supported), (
|
|
f"Expected at least {len(supported)} progress reports, "
|
|
f"got {len(progress_reports)}"
|
|
)
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_progress_reports_cover_all_resource_types(
|
|
self, provider, credentials
|
|
):
|
|
"""Progress reports must cover every resource type being scanned."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = ProgressTrackingPlugin(supported_types=supported)
|
|
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=None,
|
|
)
|
|
|
|
progress_reports: list[ScanProgress] = []
|
|
|
|
def track_progress(progress: ScanProgress) -> None:
|
|
progress_reports.append(progress)
|
|
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
scanner.scan(progress_callback=track_progress)
|
|
|
|
# Every resource type should appear in at least one progress report
|
|
reported_types = {p.current_resource_type for p in progress_reports}
|
|
for rt in supported:
|
|
assert rt in reported_types, (
|
|
f"Expected resource type '{rt}' in progress reports, "
|
|
f"but only found: {reported_types}"
|
|
)
|
|
|
|
|
|
class TestPartialInventoryPreservationOnFailure:
|
|
"""Property 5: Partial inventory preservation on failure.
|
|
|
|
If the Provider API connection is lost during an active scan, the Scanner
|
|
SHALL return a partial resource inventory.
|
|
|
|
**Validates: Requirements 1.7**
|
|
"""
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
fail_after=st.integers(min_value=0, max_value=10),
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_connection_loss_raises_with_partial_result(
|
|
self, provider, credentials, fail_after
|
|
):
|
|
"""Connection loss must raise ConnectionLostError with partial result."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = ConnectionLossPlugin(
|
|
supported_types=supported,
|
|
fail_after=fail_after,
|
|
)
|
|
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=None,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
|
|
with pytest.raises(ConnectionLostError) as exc_info:
|
|
scanner.scan()
|
|
|
|
error = exc_info.value
|
|
# Must have a partial_result attribute
|
|
assert hasattr(error, "partial_result")
|
|
partial = error.partial_result
|
|
assert isinstance(partial, ScanResult)
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
fail_after=st.integers(min_value=0, max_value=10),
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_partial_result_is_marked_as_partial(
|
|
self, provider, credentials, fail_after
|
|
):
|
|
"""Partial result from connection loss must have is_partial=True."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = ConnectionLossPlugin(
|
|
supported_types=supported,
|
|
fail_after=fail_after,
|
|
)
|
|
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=None,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
|
|
with pytest.raises(ConnectionLostError) as exc_info:
|
|
scanner.scan()
|
|
|
|
partial = exc_info.value.partial_result
|
|
assert partial.is_partial is True, (
|
|
"Partial result from connection loss must have is_partial=True"
|
|
)
|
|
|
|
@given(
|
|
provider=provider_type_strategy,
|
|
credentials=non_empty_credentials_strategy,
|
|
fail_after=st.integers(min_value=0, max_value=10),
|
|
)
|
|
@settings(max_examples=100)
|
|
def test_partial_result_contains_error_info(
|
|
self, provider, credentials, fail_after
|
|
):
|
|
"""Partial result must contain error information about the failure."""
|
|
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
|
plugin = ConnectionLossPlugin(
|
|
supported_types=supported,
|
|
fail_after=fail_after,
|
|
)
|
|
|
|
profile = ScanProfile(
|
|
provider=provider,
|
|
credentials=credentials,
|
|
resource_type_filters=None,
|
|
)
|
|
scanner = Scanner(profile=profile, plugin=plugin)
|
|
|
|
with pytest.raises(ConnectionLostError) as exc_info:
|
|
scanner.scan()
|
|
|
|
partial = exc_info.value.partial_result
|
|
# Must have at least one error or warning indicating the failure
|
|
assert len(partial.errors) > 0 or len(partial.warnings) > 0, (
|
|
"Partial result must contain error/warning info about the connection loss"
|
|
)
|