ops-bridge/tests/test_coverage_completeness.py

"""Cross-mode capability coverage meta-test.

Enforces that every capability in the registry has at least one test
marked with @pytest.mark.capability(name) and @pytest.mark.access_mode(mode)
for each of its required_access_modes.

The test discovers coverage by walking all collected test items, so it will
only pass when the full test suite is collected (i.e. run without -k filters
that exclude capability-marked tests).

Also validates the registry itself is self-consistent.
"""
from __future__ import annotations

import pytest

from bridge.capabilities import CAPABILITIES, CAPABILITIES_BY_NAME
from tests.conftest import collect_capability_coverage


# ---------------------------------------------------------------------------
# Registry self-consistency
# ---------------------------------------------------------------------------

def test_registry_has_capabilities():
    """Sanity: registry must be non-empty."""
    assert len(CAPABILITIES) > 0


def test_registry_names_are_unique():
    names = [c.name for c in CAPABILITIES]
    assert len(names) == len(set(names)), "Duplicate capability names in registry"


def test_registry_access_modes_are_valid():
    valid = {"cli", "mcp", "skill"}
    for cap in CAPABILITIES:
        unknown = cap.required_access_modes - valid
        assert not unknown, (
            f"Capability '{cap.name}' has unknown access modes: {unknown}"
        )


def test_registry_each_capability_has_at_least_one_mode():
    for cap in CAPABILITIES:
        assert cap.required_access_modes, (
            f"Capability '{cap.name}' has no required_access_modes"
        )


# ---------------------------------------------------------------------------
# Cross-mode coverage completeness (session-scope fixture)
# ---------------------------------------------------------------------------

@pytest.fixture(scope="session")
def capability_coverage(request) -> set[tuple[str, str]]:
    """Collect all (capability, access_mode) pairs from the test session."""
    return collect_capability_coverage(request.session.items)


def test_all_required_modes_have_tests(capability_coverage):
    """Every (capability, mode) pair in the registry must have a test."""
    missing: list[str] = []
    for cap in CAPABILITIES:
        for mode in sorted(cap.required_access_modes):
            if (cap.name, mode) not in capability_coverage:
                missing.append(f"  {cap.name!r} × {mode!r}")

    if missing:
        pytest.fail(
            "Missing test coverage for the following (capability, access_mode) pairs:\n"
            + "\n".join(missing)
            + "\n\nAdd a test with @pytest.mark.capability(<name>) and "
            "@pytest.mark.access_mode(<mode>)."
        )


# ---------------------------------------------------------------------------
# T02 — Registry completeness against CLI commands and MCP tools
# ---------------------------------------------------------------------------

def test_registry_cli_capabilities_have_matching_commands():
    """Every capability requiring CLI must have a corresponding CLI command.

    Checks that the registry doesn't list CLI requirements for operations that
    don't actually exist as CLI commands. Uses the Typer app's callback names.
    """
    from bridge.cli import app, targets_app, catalog_app

    # Collect all CLI callback function names (canonical command identity)
    top_level = {f"bridge_{cmd.callback.__name__}" for cmd in app.registered_commands}
    # targets sub-commands: callback name "targets_show" → "catalog_show_target"
    targets_cmds = set()
    for cmd in targets_app.registered_commands:
        fn = cmd.callback.__name__
        if fn == "targets_show":
            targets_cmds.add("catalog_show_target")
    catalog_cmds = set()
    for cmd in catalog_app.registered_commands:
        fn = cmd.callback.__name__
        if fn == "catalog_list":
            catalog_cmds.add("catalog_list_domains")
        elif fn == "catalog_validate":
            catalog_cmds.add("catalog_validate")
        elif fn == "catalog_show":
            catalog_cmds.add("catalog_show_bridge")

    # Also include catalog_list_targets (from targets_app without sub-command filter)
    # The targets app root command lists targets
    all_cli_caps = top_level | targets_cmds | catalog_cmds | {"catalog_list_targets"}

    for cap in CAPABILITIES:
        if "cli" in cap.required_access_modes:
            assert cap.name in all_cli_caps, (
                f"Capability '{cap.name}' requires CLI coverage but no matching "
                f"CLI command was found. Either add the command or update the registry."
            )


async def test_mcp_tools_in_registry():
    """Every MCP tool name must appear as a capability in the registry."""
    from fastmcp import Client
    from bridge.mcp_server.server import mcp

    async with Client(mcp) as c:
        tools = await c.list_tools()
    tool_names = {t.name for t in tools}

    registered_cap_names = set(CAPABILITIES_BY_NAME)
    for name in tool_names:
        assert name in registered_cap_names, (
            f"MCP tool '{name}' is not registered as a capability. "
            f"Add it to src/bridge/capabilities.py."
        )


# ---------------------------------------------------------------------------
# T12 — Self-validation: sentinel fixture proves the gap-checker catches gaps
# ---------------------------------------------------------------------------

def test_meta_test_catches_missing_mode_gap():
    """Self-validation: the coverage checker must detect a missing-mode gap.

    Injects a synthetic _test_sentinel capability requiring both cli and mcp.
    Creates mock items with *only* a cli test for it (deliberately omitting mcp).
    Asserts that collect_capability_coverage reports the mcp gap — proving the
    meta-test mechanism is functional, not a silent no-op.

    This test validates Goal #4 from BRIDGE-WP-0003:
        "The gap-detection mechanism is itself tested: a synthetic missing-mode
        fixture asserts the meta-test catches it."
    """
    from bridge.capabilities import Capability

    sentinel = Capability(
        name="_test_sentinel",
        description="Synthetic capability for meta-test self-validation",
        required_access_modes=frozenset({"cli", "mcp"}),
    )
    patched_caps = CAPABILITIES + [sentinel]

    # Minimal mock: an iterable of items that respond to iter_markers()
    class _Mark:
        def __init__(self, arg: str):
            self.args = (arg,)

    class _MockItem:
        def __init__(self, capability: str, mode: str):
            self._cap = capability
            self._mode = mode

        def iter_markers(self, name: str):
            if name == "capability":
                return [_Mark(self._cap)]
            if name == "access_mode":
                return [_Mark(self._mode)]
            return []

    # Only supply a cli test for the sentinel — the mcp test is intentionally absent
    mock_items = [_MockItem("_test_sentinel", "cli")]

    covered = collect_capability_coverage(mock_items)

    # The cli mode should be registered
    assert ("_test_sentinel", "cli") in covered, (
        "collect_capability_coverage failed to record the cli mock item"
    )
    # The mcp mode must NOT be covered — this is the gap we want to catch
    assert ("_test_sentinel", "mcp") not in covered, (
        "collect_capability_coverage incorrectly registered an mcp test that was not provided"
    )

    # Run the same gap-detection logic used by test_all_required_modes_have_tests
    gaps = [
        (cap.name, mode)
        for cap in patched_caps
        for mode in cap.required_access_modes
        if (cap.name, mode) not in covered
    ]

    assert ("_test_sentinel", "mcp") in gaps, (
        "Gap checker failed to detect the missing mcp mode for _test_sentinel. "
        "The meta-test mechanism is broken."
    )
    # Sanity: cli mode should NOT appear as a gap (it was covered)
    assert ("_test_sentinel", "cli") not in gaps


def test_no_orphan_capability_marks(capability_coverage):
    """Every (capability, mode) pair in the test suite must exist in the registry.

    This prevents tests from referencing stale or misspelled capability names.
    """
    orphans: list[str] = []
    for cap_name, mode in sorted(capability_coverage):
        if cap_name not in CAPABILITIES_BY_NAME:
            orphans.append(f"  {cap_name!r} (mode={mode!r}) — not in registry")
        else:
            cap = CAPABILITIES_BY_NAME[cap_name]
            if mode not in cap.required_access_modes:
                orphans.append(
                    f"  {cap_name!r} × {mode!r} — mode not required for this capability"
                )

    if orphans:
        pytest.fail(
            "Test suite references capability/mode pairs not in registry:\n"
            + "\n".join(orphans)
        )