ops-bridge/tests/test_coverage_completeness.py

"""Cross-mode capability coverage meta-test.

Enforces that every capability in the registry has at least one test
marked with @pytest.mark.capability(name) and @pytest.mark.access_mode(mode)
for each of its required_access_modes.

The test discovers coverage by walking all collected test items, so it will
only pass when the full test suite is collected (i.e. run without -k filters
that exclude capability-marked tests).

Also validates the registry itself is self-consistent.
"""
from __future__ import annotations

import pytest

from bridge.capabilities import CAPABILITIES, CAPABILITIES_BY_NAME
from tests.conftest import collect_capability_coverage


# ---------------------------------------------------------------------------
# Registry self-consistency
# ---------------------------------------------------------------------------

def test_registry_has_capabilities():
    """Sanity: registry must be non-empty."""
    assert len(CAPABILITIES) > 0


def test_registry_names_are_unique():
    names = [c.name for c in CAPABILITIES]
    assert len(names) == len(set(names)), "Duplicate capability names in registry"


def test_registry_access_modes_are_valid():
    valid = {"cli", "mcp", "skill"}
    for cap in CAPABILITIES:
        unknown = cap.required_access_modes - valid
        assert not unknown, (
            f"Capability '{cap.name}' has unknown access modes: {unknown}"
        )


def test_registry_each_capability_has_at_least_one_mode():
    for cap in CAPABILITIES:
        assert cap.required_access_modes, (
            f"Capability '{cap.name}' has no required_access_modes"
        )


# ---------------------------------------------------------------------------
# Cross-mode coverage completeness (session-scope fixture)
# ---------------------------------------------------------------------------

@pytest.fixture(scope="session")
def capability_coverage(request) -> set[tuple[str, str]]:
    """Collect all (capability, access_mode) pairs from the test session."""
    return collect_capability_coverage(request.session.items)


def test_all_required_modes_have_tests(capability_coverage):
    """Every (capability, mode) pair in the registry must have a test."""
    missing: list[str] = []
    for cap in CAPABILITIES:
        for mode in sorted(cap.required_access_modes):
            if (cap.name, mode) not in capability_coverage:
                missing.append(f"  {cap.name!r} × {mode!r}")

    if missing:
        pytest.fail(
            "Missing test coverage for the following (capability, access_mode) pairs:\n"
            + "\n".join(missing)
            + "\n\nAdd a test with @pytest.mark.capability(<name>) and "
            "@pytest.mark.access_mode(<mode>)."
        )


# ---------------------------------------------------------------------------
# T02 — Registry completeness against CLI commands and MCP tools
# ---------------------------------------------------------------------------

def test_registry_cli_capabilities_have_matching_commands():
    """Every capability requiring CLI must have a corresponding CLI command.

    Checks that the registry doesn't list CLI requirements for operations that
    don't actually exist as CLI commands. Uses the Typer app's callback names.
    """
    from bridge.cli import app, targets_app, catalog_app

    # Collect all CLI callback function names (canonical command identity)
    top_level = {f"bridge_{cmd.callback.__name__}" for cmd in app.registered_commands}
    # targets sub-commands: callback name "targets_show" → "catalog_show_target"
    targets_cmds = set()
    for cmd in targets_app.registered_commands:
        fn = cmd.callback.__name__
        if fn == "targets_show":
            targets_cmds.add("catalog_show_target")
    catalog_cmds = set()
    for cmd in catalog_app.registered_commands:
        fn = cmd.callback.__name__
        if fn == "catalog_list":
            catalog_cmds.add("catalog_list_domains")
        elif fn == "catalog_validate":
            catalog_cmds.add("catalog_validate")
        elif fn == "catalog_show":
            catalog_cmds.add("catalog_show_bridge")

    # Also include catalog_list_targets (from targets_app without sub-command filter)
    # The targets app root command lists targets
    all_cli_caps = top_level | targets_cmds | catalog_cmds | {"catalog_list_targets"}

    for cap in CAPABILITIES:
        if "cli" in cap.required_access_modes:
            assert cap.name in all_cli_caps, (
                f"Capability '{cap.name}' requires CLI coverage but no matching "
                f"CLI command was found. Either add the command or update the registry."
            )


async def test_mcp_tools_in_registry():
    """Every MCP tool name must appear as a capability in the registry."""
    from fastmcp import Client
    from bridge.mcp_server.server import mcp

    async with Client(mcp) as c:
        tools = await c.list_tools()
    tool_names = {t.name for t in tools}

    registered_cap_names = set(CAPABILITIES_BY_NAME)
    for name in tool_names:
        assert name in registered_cap_names, (
            f"MCP tool '{name}' is not registered as a capability. "
            f"Add it to src/bridge/capabilities.py."
        )


def test_no_orphan_capability_marks(capability_coverage):
    """Every (capability, mode) pair in the test suite must exist in the registry.

    This prevents tests from referencing stale or misspelled capability names.
    """
    orphans: list[str] = []
    for cap_name, mode in sorted(capability_coverage):
        if cap_name not in CAPABILITIES_BY_NAME:
            orphans.append(f"  {cap_name!r} (mode={mode!r}) — not in registry")
        else:
            cap = CAPABILITIES_BY_NAME[cap_name]
            if mode not in cap.required_access_modes:
                orphans.append(
                    f"  {cap_name!r} × {mode!r} — mode not required for this capability"
                )

    if orphans:
        pytest.fail(
            "Test suite references capability/mode pairs not in registry:\n"
            + "\n".join(orphans)
        )