Created IAC reverse generator
This commit is contained in:
p2913020
257
tests/property/test_scan_profile_validation_prop.py
Normal file
257
tests/property/test_scan_profile_validation_prop.py
Normal file
@@ -0,0 +1,257 @@
|
||||
"""Property-based tests for ScanProfile validation completeness.
|
||||
|
||||
**Validates: Requirements 6.1, 6.6, 6.7**
|
||||
|
||||
Property 20: Scan profile validation completeness
|
||||
For any scan profile with K invalid fields (missing provider, empty credentials,
|
||||
unreachable endpoints, filters exceeding 200 entries, or unsupported resource types),
|
||||
the validation error SHALL list all K invalid fields in a single response.
|
||||
"""
|
||||
|
||||
from hypothesis import given, assume, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from iac_reverse.models import (
|
||||
MAX_RESOURCE_TYPE_FILTERS,
|
||||
PROVIDER_SUPPORTED_RESOURCE_TYPES,
|
||||
ProviderType,
|
||||
ScanProfile,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hypothesis Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
provider_type_strategy = st.sampled_from(list(ProviderType))
|
||||
|
||||
non_empty_credentials_strategy = st.dictionaries(
|
||||
keys=st.text(min_size=1, max_size=20, alphabet=st.characters(whitelist_categories=("L", "N", "P"))),
|
||||
values=st.text(min_size=1, max_size=50),
|
||||
min_size=1,
|
||||
max_size=5,
|
||||
)
|
||||
|
||||
empty_credentials_strategy = st.just({})
|
||||
|
||||
|
||||
def valid_resource_types_strategy(provider: ProviderType) -> st.SearchStrategy:
|
||||
"""Generate a list of valid resource types for the given provider."""
|
||||
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
||||
return st.lists(st.sampled_from(supported), min_size=0, max_size=min(len(supported), 10))
|
||||
|
||||
|
||||
invalid_resource_type_strategy = st.text(
|
||||
min_size=5, max_size=30,
|
||||
alphabet=st.characters(whitelist_categories=("L",))
|
||||
).filter(
|
||||
lambda t: all(t not in types for types in PROVIDER_SUPPORTED_RESOURCE_TYPES.values())
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestScanProfileValidationCompleteness:
|
||||
"""Property 20: Scan profile validation completeness.
|
||||
|
||||
**Validates: Requirements 6.1, 6.6, 6.7**
|
||||
"""
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
credentials=non_empty_credentials_strategy,
|
||||
)
|
||||
def test_valid_profile_returns_no_errors(self, provider, credentials):
|
||||
"""A profile with non-empty credentials and no filters is always valid."""
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials=credentials,
|
||||
resource_type_filters=None,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert errors == [], f"Expected no errors for valid profile, got: {errors}"
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
credentials=non_empty_credentials_strategy,
|
||||
)
|
||||
def test_valid_profile_with_valid_filters_returns_no_errors(self, provider, credentials):
|
||||
"""A profile with valid credentials and valid resource type filters is valid."""
|
||||
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials=credentials,
|
||||
resource_type_filters=supported,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert errors == [], f"Expected no errors for valid profile with valid filters, got: {errors}"
|
||||
|
||||
@given(provider=provider_type_strategy)
|
||||
def test_empty_credentials_always_produces_credentials_error(self, provider):
|
||||
"""Empty credentials must always produce an error mentioning 'credentials'."""
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials={},
|
||||
resource_type_filters=None,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert len(errors) >= 1
|
||||
assert any("credentials" in e for e in errors), (
|
||||
f"Expected error mentioning 'credentials', got: {errors}"
|
||||
)
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
credentials=non_empty_credentials_strategy,
|
||||
extra_count=st.integers(min_value=1, max_value=50),
|
||||
)
|
||||
def test_oversized_filters_produces_count_error(self, provider, credentials, extra_count):
|
||||
"""Filters exceeding MAX_RESOURCE_TYPE_FILTERS must produce an error about the count limit."""
|
||||
# Build a list that exceeds the limit using valid types repeated
|
||||
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
||||
oversized_count = MAX_RESOURCE_TYPE_FILTERS + extra_count
|
||||
filters = (supported * (oversized_count // len(supported) + 1))[:oversized_count]
|
||||
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials=credentials,
|
||||
resource_type_filters=filters,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert any(
|
||||
"at most" in e or str(MAX_RESOURCE_TYPE_FILTERS) in e
|
||||
for e in errors
|
||||
), f"Expected error mentioning count limit, got: {errors}"
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
credentials=non_empty_credentials_strategy,
|
||||
invalid_types=st.lists(invalid_resource_type_strategy, min_size=1, max_size=5),
|
||||
)
|
||||
def test_unsupported_types_produces_unsupported_error(self, provider, credentials, invalid_types):
|
||||
"""Unsupported resource types must produce an error mentioning them."""
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials=credentials,
|
||||
resource_type_filters=invalid_types,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert any("unsupported" in e.lower() for e in errors), (
|
||||
f"Expected error mentioning unsupported types, got: {errors}"
|
||||
)
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
invalid_types=st.lists(invalid_resource_type_strategy, min_size=1, max_size=3),
|
||||
)
|
||||
def test_no_short_circuit_credentials_and_unsupported(self, provider, invalid_types):
|
||||
"""When both credentials are empty AND unsupported types exist, both errors are reported."""
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials={},
|
||||
resource_type_filters=invalid_types,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert len(errors) >= 2, f"Expected at least 2 errors, got {len(errors)}: {errors}"
|
||||
assert any("credentials" in e for e in errors), (
|
||||
f"Expected credentials error, got: {errors}"
|
||||
)
|
||||
assert any("unsupported" in e.lower() for e in errors), (
|
||||
f"Expected unsupported types error, got: {errors}"
|
||||
)
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
extra_count=st.integers(min_value=1, max_value=20),
|
||||
)
|
||||
def test_no_short_circuit_credentials_and_oversized(self, provider, extra_count):
|
||||
"""When both credentials are empty AND filters exceed limit, both errors are reported."""
|
||||
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
||||
oversized_count = MAX_RESOURCE_TYPE_FILTERS + extra_count
|
||||
filters = (supported * (oversized_count // len(supported) + 1))[:oversized_count]
|
||||
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials={},
|
||||
resource_type_filters=filters,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert len(errors) >= 2, f"Expected at least 2 errors, got {len(errors)}: {errors}"
|
||||
assert any("credentials" in e for e in errors), (
|
||||
f"Expected credentials error, got: {errors}"
|
||||
)
|
||||
assert any(
|
||||
"at most" in e or str(MAX_RESOURCE_TYPE_FILTERS) in e
|
||||
for e in errors
|
||||
), f"Expected count limit error, got: {errors}"
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
extra_count=st.integers(min_value=1, max_value=10),
|
||||
invalid_types=st.lists(invalid_resource_type_strategy, min_size=1, max_size=3),
|
||||
)
|
||||
def test_no_short_circuit_all_three_issues(self, provider, extra_count, invalid_types):
|
||||
"""When credentials empty, filters oversized, AND unsupported types exist, all errors reported."""
|
||||
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
||||
oversized_count = MAX_RESOURCE_TYPE_FILTERS + extra_count
|
||||
# Mix valid types (to reach oversized count) with invalid types
|
||||
valid_padding = (supported * (oversized_count // len(supported) + 1))[:oversized_count]
|
||||
filters = valid_padding + invalid_types
|
||||
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials={},
|
||||
resource_type_filters=filters,
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert len(errors) >= 3, f"Expected at least 3 errors, got {len(errors)}: {errors}"
|
||||
assert any("credentials" in e for e in errors), (
|
||||
f"Expected credentials error, got: {errors}"
|
||||
)
|
||||
assert any(
|
||||
"at most" in e or str(MAX_RESOURCE_TYPE_FILTERS) in e
|
||||
for e in errors
|
||||
), f"Expected count limit error, got: {errors}"
|
||||
assert any("unsupported" in e.lower() for e in errors), (
|
||||
f"Expected unsupported types error, got: {errors}"
|
||||
)
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
credentials=non_empty_credentials_strategy,
|
||||
)
|
||||
def test_empty_list_filters_is_valid(self, provider, credentials):
|
||||
"""An empty resource_type_filters list (not None) should be valid."""
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials=credentials,
|
||||
resource_type_filters=[],
|
||||
)
|
||||
errors = profile.validate()
|
||||
assert errors == [], f"Expected no errors for empty filter list, got: {errors}"
|
||||
|
||||
@given(
|
||||
provider=provider_type_strategy,
|
||||
credentials=non_empty_credentials_strategy,
|
||||
count=st.integers(min_value=1, max_value=MAX_RESOURCE_TYPE_FILTERS),
|
||||
)
|
||||
def test_filters_at_or_below_limit_with_valid_types_is_valid(self, provider, credentials, count):
|
||||
"""Any number of valid filters at or below the limit should produce no count error."""
|
||||
supported = PROVIDER_SUPPORTED_RESOURCE_TYPES[provider]
|
||||
# Repeat valid types to reach the desired count
|
||||
filters = (supported * (count // len(supported) + 1))[:count]
|
||||
|
||||
profile = ScanProfile(
|
||||
provider=provider,
|
||||
credentials=credentials,
|
||||
resource_type_filters=filters,
|
||||
)
|
||||
errors = profile.validate()
|
||||
# Should not have a count limit error
|
||||
assert not any(
|
||||
"at most" in e or (str(MAX_RESOURCE_TYPE_FILTERS) in e and "entries" in e)
|
||||
for e in errors
|
||||
), f"Unexpected count limit error for {count} filters: {errors}"
|
||||
Reference in New Issue
Block a user