tegwick
bd1d05ba79
Add change detection, structural diff-based impact analysis, configurable-depth incremental recomputation with circular suppression, and impact debt tracking. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
290 lines
9.7 KiB
Python
290 lines
9.7 KiB
Python
"""
|
|
Integration tests for impact debt tracking.
|
|
|
|
Tests below-threshold suppression, budget exhaustion, and debt querying
|
|
with a real SQLite database.
|
|
"""
|
|
|
|
import pytest
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from markitect.prompts.models import Artifact, ArtifactType
|
|
from markitect.prompts.repositories.sqlite import SQLiteArtifactRepository
|
|
from markitect.prompts.dependencies.models import DependencyEdge, EdgeType
|
|
from markitect.prompts.dependencies.repository import SQLiteDependencyRepository
|
|
from markitect.prompts.dependencies.queries import DependencyQueryService
|
|
from markitect.prompts.incremental.detector import ChangeDetector
|
|
from markitect.prompts.incremental.engine import IncrementalExecutionEngine
|
|
from markitect.prompts.incremental.models import RecomputeConfig
|
|
|
|
|
|
@pytest.fixture
|
|
def temp_db():
|
|
"""Create temporary database for testing."""
|
|
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
|
|
db_path = f.name
|
|
yield db_path
|
|
Path(db_path).unlink(missing_ok=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def artifact_repo(temp_db):
|
|
"""Create artifact repository."""
|
|
return SQLiteArtifactRepository(temp_db)
|
|
|
|
|
|
@pytest.fixture
|
|
def dep_repo(temp_db):
|
|
"""Create dependency repository."""
|
|
return SQLiteDependencyRepository(temp_db)
|
|
|
|
|
|
@pytest.fixture
|
|
def query_service(dep_repo):
|
|
"""Create DependencyQueryService."""
|
|
return DependencyQueryService(dep_repo)
|
|
|
|
|
|
@pytest.fixture
|
|
def detector(temp_db):
|
|
"""Create ChangeDetector."""
|
|
return ChangeDetector(temp_db)
|
|
|
|
|
|
@pytest.fixture
|
|
def engine(temp_db, query_service):
|
|
"""Create IncrementalExecutionEngine."""
|
|
return IncrementalExecutionEngine(temp_db, query_service)
|
|
|
|
|
|
def _create_artifact(repo, space_id, name, content="content"):
|
|
"""Helper to create and persist an artifact."""
|
|
artifact = Artifact.create(
|
|
space_id=space_id,
|
|
name=name,
|
|
content=content,
|
|
artifact_type=ArtifactType.CONTENT,
|
|
)
|
|
return repo.create(artifact)
|
|
|
|
|
|
def _create_edge(repo, src, tgt, run_id="run-1"):
|
|
"""Helper to create and persist a dependency edge."""
|
|
edge = DependencyEdge.create(
|
|
source_artifact_id=src,
|
|
target_artifact_id=tgt,
|
|
run_id=run_id,
|
|
edge_type=EdgeType.REQUIRES,
|
|
)
|
|
return repo.create(edge)
|
|
|
|
|
|
class TestBelowThresholdSuppression:
|
|
"""Tests for impact debt from below-threshold suppression."""
|
|
|
|
def test_small_change_creates_debt(
|
|
self, artifact_repo, dep_repo, detector, engine
|
|
):
|
|
"""Test small change below threshold creates impact debt."""
|
|
lib = _create_artifact(artifact_repo, "space-1", "lib", "hello world")
|
|
app = _create_artifact(artifact_repo, "space-1", "app", "uses lib")
|
|
|
|
_create_edge(dep_repo, app.id, lib.id)
|
|
|
|
change = detector.detect_change(lib, "hello World") # tiny change
|
|
assert change is not None
|
|
detector.record_change(change)
|
|
|
|
result = engine.recompute(
|
|
change,
|
|
config=RecomputeConfig(max_depth=1, impact_threshold=0.5),
|
|
old_content="hello world",
|
|
new_content="hello World",
|
|
)
|
|
|
|
assert result.suppressed_count == 1
|
|
assert result.recomputed_count == 0
|
|
assert result.suppressed[0].suppression_reason == "below_threshold"
|
|
|
|
def test_debt_records_magnitude(
|
|
self, artifact_repo, dep_repo, detector, engine
|
|
):
|
|
"""Test debt records include change magnitude."""
|
|
lib = _create_artifact(artifact_repo, "space-1", "lib", "content A")
|
|
app = _create_artifact(artifact_repo, "space-1", "app", "uses lib")
|
|
|
|
_create_edge(dep_repo, app.id, lib.id)
|
|
|
|
change = detector.detect_change(lib, "content B")
|
|
detector.record_change(change)
|
|
|
|
engine.recompute(
|
|
change,
|
|
config=RecomputeConfig(max_depth=1, impact_threshold=0.9),
|
|
old_content="content A",
|
|
new_content="content B",
|
|
)
|
|
|
|
debt = engine.get_debt_for_artifact(lib.id)
|
|
assert len(debt) == 1
|
|
assert debt[0].change_magnitude > 0.0
|
|
assert debt[0].change_magnitude < 1.0
|
|
|
|
def test_large_change_no_debt(
|
|
self, artifact_repo, dep_repo, detector, engine
|
|
):
|
|
"""Test large change above threshold creates no debt."""
|
|
lib = _create_artifact(artifact_repo, "space-1", "lib", "old content here")
|
|
app = _create_artifact(artifact_repo, "space-1", "app", "uses lib")
|
|
|
|
_create_edge(dep_repo, app.id, lib.id)
|
|
|
|
change = detector.detect_change(lib, "completely new different content xyz")
|
|
detector.record_change(change)
|
|
|
|
result = engine.recompute(
|
|
change,
|
|
config=RecomputeConfig(max_depth=1, impact_threshold=0.1),
|
|
old_content="old content here",
|
|
new_content="completely new different content xyz",
|
|
)
|
|
|
|
assert result.recomputed_count == 1
|
|
assert result.suppressed_count == 0
|
|
|
|
debt = engine.get_debt_for_artifact(lib.id)
|
|
assert len(debt) == 0
|
|
|
|
|
|
class TestBudgetExhaustion:
|
|
"""Tests for impact debt from budget exhaustion."""
|
|
|
|
def test_budget_creates_debt_for_excess(
|
|
self, artifact_repo, dep_repo, detector, engine
|
|
):
|
|
"""Test budget exhaustion creates debt for overflow dependents."""
|
|
lib = _create_artifact(artifact_repo, "space-1", "lib", "lib v1")
|
|
|
|
# Create 5 apps depending on lib
|
|
apps = []
|
|
for i in range(5):
|
|
app = _create_artifact(artifact_repo, "space-1", f"app-{i}", f"app-{i}")
|
|
_create_edge(dep_repo, app.id, lib.id, run_id=f"run-{i}")
|
|
apps.append(app)
|
|
|
|
change = detector.detect_change(lib, "lib v2")
|
|
detector.record_change(change)
|
|
|
|
result = engine.recompute(
|
|
change,
|
|
config=RecomputeConfig(max_depth=1, max_recomputes=2),
|
|
old_content="lib v1",
|
|
new_content="lib v2",
|
|
)
|
|
|
|
assert result.total_dependents == 5
|
|
assert result.recomputed_count == 2
|
|
assert result.suppressed_count == 3
|
|
|
|
budget_debt = [d for d in result.suppressed if d.suppression_reason == "budget_exhausted"]
|
|
assert len(budget_debt) == 3
|
|
|
|
def test_budget_debt_queryable(
|
|
self, artifact_repo, dep_repo, detector, engine
|
|
):
|
|
"""Test budget-exhaustion debt is queryable from DB."""
|
|
lib = _create_artifact(artifact_repo, "space-1", "lib", "lib v1")
|
|
|
|
for i in range(3):
|
|
app = _create_artifact(artifact_repo, "space-1", f"app-{i}", f"app-{i}")
|
|
_create_edge(dep_repo, app.id, lib.id, run_id=f"run-{i}")
|
|
|
|
change = detector.detect_change(lib, "lib v2")
|
|
detector.record_change(change)
|
|
|
|
engine.recompute(
|
|
change,
|
|
config=RecomputeConfig(max_depth=1, max_recomputes=1),
|
|
old_content="lib v1",
|
|
new_content="lib v2",
|
|
)
|
|
|
|
all_debt = engine.get_all_debt()
|
|
budget_debt = [d for d in all_debt if d.suppression_reason == "budget_exhausted"]
|
|
assert len(budget_debt) == 2
|
|
|
|
|
|
class TestDebtQuerying:
|
|
"""Tests for querying impact debt records."""
|
|
|
|
def test_query_by_artifact(
|
|
self, artifact_repo, dep_repo, detector, engine
|
|
):
|
|
"""Test querying debt by artifact ID."""
|
|
lib_a = _create_artifact(artifact_repo, "space-1", "lib-a", "a-v1")
|
|
lib_b = _create_artifact(artifact_repo, "space-1", "lib-b", "b-v1")
|
|
app = _create_artifact(artifact_repo, "space-1", "app", "app")
|
|
|
|
_create_edge(dep_repo, app.id, lib_a.id)
|
|
_create_edge(dep_repo, app.id, lib_b.id)
|
|
|
|
# Suppress change to lib_a
|
|
change_a = detector.detect_change(lib_a, "a-v2")
|
|
detector.record_change(change_a)
|
|
engine.recompute(
|
|
change_a,
|
|
config=RecomputeConfig(max_depth=1, impact_threshold=0.99),
|
|
old_content="a-v1",
|
|
new_content="a-v2",
|
|
)
|
|
|
|
# Suppress change to lib_b
|
|
change_b = detector.detect_change(lib_b, "b-v2")
|
|
detector.record_change(change_b)
|
|
engine.recompute(
|
|
change_b,
|
|
config=RecomputeConfig(max_depth=1, impact_threshold=0.99),
|
|
old_content="b-v1",
|
|
new_content="b-v2",
|
|
)
|
|
|
|
# Query by artifact
|
|
debt_a = engine.get_debt_for_artifact(lib_a.id)
|
|
assert len(debt_a) == 1
|
|
|
|
debt_b = engine.get_debt_for_artifact(lib_b.id)
|
|
assert len(debt_b) == 1
|
|
|
|
# Total debt
|
|
all_debt = engine.get_all_debt()
|
|
assert len(all_debt) == 2
|
|
|
|
def test_query_by_run(
|
|
self, artifact_repo, dep_repo, detector, engine
|
|
):
|
|
"""Test querying debt by dependent run ID."""
|
|
lib = _create_artifact(artifact_repo, "space-1", "lib", "lib-v1")
|
|
app = _create_artifact(artifact_repo, "space-1", "app", "app")
|
|
|
|
_create_edge(dep_repo, app.id, lib.id)
|
|
|
|
change = detector.detect_change(lib, "lib-v2")
|
|
detector.record_change(change)
|
|
engine.recompute(
|
|
change,
|
|
config=RecomputeConfig(max_depth=1, impact_threshold=0.99),
|
|
old_content="lib-v1",
|
|
new_content="lib-v2",
|
|
)
|
|
|
|
debt = engine.get_debt_for_run(app.id)
|
|
assert len(debt) == 1
|
|
assert debt[0].dependent_run_id == app.id
|
|
|
|
def test_no_debt_returns_empty(self, engine):
|
|
"""Test querying debt when none exists returns empty list."""
|
|
assert engine.get_debt_for_artifact("nonexistent") == []
|
|
assert engine.get_debt_for_run("nonexistent") == []
|
|
assert engine.get_all_debt() == []
|