From 0192dc786f5b527de4f9bc0473a207191bd12307 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 19 Jun 2026 20:46:07 +0200 Subject: [PATCH] STATE-WP-0062 T1: two-dimension service_catalog model + migration Add ServiceCatalog core (hosting_type, development_type, maturity_level) plus 1:1 per-dimension extension tables (service_third_party, service_first_party, service_cloud, service_self_hosted) keyed by service_id. Migration creates the tables and copies existing tpsc_catalog rows into service_catalog as (cloud_hosted, third_party), reusing the tpsc_catalog id as the service_catalog id so existing tpsc_entries.catalog_id keep resolving without a column change. GDPR/data-processing fields move to service_cloud; pricing_model to service_third_party. Co-Authored-By: Claude Opus 4.8 --- api/models/__init__.py | 9 ++ api/models/service_catalog.py | 121 ++++++++++++++++ .../versions/c7d8e9f0a1b2_service_catalog.py | 131 ++++++++++++++++++ 3 files changed, 261 insertions(+) create mode 100644 api/models/service_catalog.py create mode 100644 migrations/versions/c7d8e9f0a1b2_service_catalog.py diff --git a/api/models/__init__.py b/api/models/__init__.py index 57b465b..6c9c3fc 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -18,6 +18,13 @@ from api.models.agent_message import AgentMessage from api.models.capability_catalog import CapabilityCatalog from api.models.capability_request import CapabilityRequest from api.models.tpsc import TPSCCatalog, TPSCSnapshot, TPSCEntry +from api.models.service_catalog import ( + ServiceCatalog, + ServiceThirdParty, + ServiceFirstParty, + ServiceCloud, + ServiceSelfHosted, +) from api.models.doi_cache import DOICache from api.models.token_event import TokenEvent from api.models.interface_change import InterfaceChange @@ -46,6 +53,8 @@ __all__ = [ "CapabilityCatalog", "CapabilityRequest", "TPSCCatalog", "TPSCSnapshot", "TPSCEntry", + "ServiceCatalog", "ServiceThirdParty", "ServiceFirstParty", + "ServiceCloud", "ServiceSelfHosted", "DOICache", "TokenEvent", "InterfaceChange", diff --git a/api/models/service_catalog.py b/api/models/service_catalog.py new file mode 100644 index 0000000..dd1da88 --- /dev/null +++ b/api/models/service_catalog.py @@ -0,0 +1,121 @@ +"""Two-dimension service catalog (STATE-WP-0062). + +Every service is classified along two orthogonal dimensions: + + - hosting_type: self_hosted (coulomb operates it) | cloud_hosted (consumed) + - development_type: first_party (coulomb develops it) | third_party (external) + +Common fields live in ``ServiceCatalog``; dimension-specific data composes via +1:1 extension tables (``service_id`` is both PK and FK), so a self-hosted +first-party service carries the self-hosted *and* first-party extensions without +needing a bespoke per-class shape. +""" +import uuid +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSON, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from api.models.base import Base + + +class ServiceCatalog(Base): + __tablename__ = "service_catalog" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + slug: Mapped[str] = mapped_column(String(100), nullable=False, unique=True, index=True) + name: Mapped[str] = mapped_column(String(200), nullable=False) + owner_or_provider: Mapped[str | None] = mapped_column(String(200), nullable=True) + category: Mapped[str | None] = mapped_column(String(100), nullable=True) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + website_url: Mapped[str | None] = mapped_column(Text, nullable=True) + # status: active | deprecated + status: Mapped[str] = mapped_column(String(20), nullable=False, server_default="active") + # hosting_type: self_hosted | cloud_hosted + hosting_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) + # development_type: first_party | third_party + development_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) + # Service DoM Level (1=Operable, 2=Observable, 3=Mature); NULL = unassessed + maturity_level: Mapped[int | None] = mapped_column(Integer, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + third_party: Mapped["ServiceThirdParty | None"] = relationship( + back_populates="service", uselist=False, cascade="all, delete-orphan") + first_party: Mapped["ServiceFirstParty | None"] = relationship( + back_populates="service", uselist=False, cascade="all, delete-orphan") + cloud: Mapped["ServiceCloud | None"] = relationship( + back_populates="service", uselist=False, cascade="all, delete-orphan") + self_hosted: Mapped["ServiceSelfHosted | None"] = relationship( + back_populates="service", uselist=False, cascade="all, delete-orphan") + + +class ServiceThirdParty(Base): + """Extension for development_type = third_party (coulomb is not dev-responsible).""" + __tablename__ = "service_third_party" + + service_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True) + # pricing_model: free | paid | freemium | usage_based | unknown + pricing_model: Mapped[str] = mapped_column(String(20), nullable=False, server_default="unknown") + upstream_packages: Mapped[list | None] = mapped_column(JSON, nullable=True) + upstream_contacts: Mapped[list | None] = mapped_column(JSON, nullable=True) + source_url: Mapped[str | None] = mapped_column(Text, nullable=True) + support_url: Mapped[str | None] = mapped_column(Text, nullable=True) + license: Mapped[str | None] = mapped_column(String(100), nullable=True) + + service: Mapped["ServiceCatalog"] = relationship(back_populates="third_party") + + +class ServiceFirstParty(Base): + """Extension for development_type = first_party (coulomb develops it).""" + __tablename__ = "service_first_party" + + service_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True) + repo_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("managed_repos.id", ondelete="SET NULL"), nullable=True, index=True) + owning_domain: Mapped[str | None] = mapped_column(String(100), nullable=True) + + service: Mapped["ServiceCatalog"] = relationship(back_populates="first_party") + + +class ServiceCloud(Base): + """Extension for hosting_type = cloud_hosted (data is processed off coulomb infra). + + Holds the data-processor concerns that were the heart of the old TPSC record; + they apply whenever data leaves coulomb infra, independent of who built it. + """ + __tablename__ = "service_cloud" + + service_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True) + # gdpr_maturity (CNIL/IAPP CMMI-aligned): + # unknown | non_compliant | initial | developing | defined | managed | certified + gdpr_maturity: Mapped[str] = mapped_column(String(20), nullable=False, server_default="unknown", index=True) + gdpr_notes: Mapped[str | None] = mapped_column(Text, nullable=True) + dpa_available: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="false") + tos_url: Mapped[str | None] = mapped_column(Text, nullable=True) + privacy_policy_url: Mapped[str | None] = mapped_column(Text, nullable=True) + data_processing_regions: Mapped[list | None] = mapped_column(JSON, nullable=True) + data_retention_notes: Mapped[str | None] = mapped_column(Text, nullable=True) + + service: Mapped["ServiceCatalog"] = relationship(back_populates="cloud") + + +class ServiceSelfHosted(Base): + """Extension for hosting_type = self_hosted (coulomb operates the service).""" + __tablename__ = "service_self_hosted" + + service_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True) + # three-helix instance / host the service runs on + helix_instance: Mapped[str | None] = mapped_column(String(100), nullable=True) + host_node: Mapped[str | None] = mapped_column(String(100), nullable=True) + deployment_ref: Mapped[str | None] = mapped_column(Text, nullable=True) + runbook_ref: Mapped[str | None] = mapped_column(Text, nullable=True) + # upstream OSS project when the self-hosted service is third-party software + upstream_oss_project: Mapped[str | None] = mapped_column(String(200), nullable=True) + + service: Mapped["ServiceCatalog"] = relationship(back_populates="self_hosted") diff --git a/migrations/versions/c7d8e9f0a1b2_service_catalog.py b/migrations/versions/c7d8e9f0a1b2_service_catalog.py new file mode 100644 index 0000000..06b5f00 --- /dev/null +++ b/migrations/versions/c7d8e9f0a1b2_service_catalog.py @@ -0,0 +1,131 @@ +"""two-dimension service catalog (STATE-WP-0062) + +Creates service_catalog + per-dimension extension tables and migrates existing +tpsc_catalog rows into it as (cloud_hosted, third_party). The service_catalog id +reuses the tpsc_catalog id so existing tpsc_entries.catalog_id values continue to +resolve against the new core table without a column change. + +Revision ID: c7d8e9f0a1b2 +Revises: 5733434addf4 +Create Date: 2026-06-19 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSON, UUID + +revision = "c7d8e9f0a1b2" +down_revision = "5733434addf4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "service_catalog", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("slug", sa.String(length=100), nullable=False), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column("owner_or_provider", sa.String(length=200), nullable=True), + sa.Column("category", sa.String(length=100), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("website_url", sa.Text(), nullable=True), + sa.Column("status", sa.String(length=20), server_default="active", nullable=False), + sa.Column("hosting_type", sa.String(length=20), nullable=False), + sa.Column("development_type", sa.String(length=20), nullable=False), + sa.Column("maturity_level", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.UniqueConstraint("slug", name="uq_service_catalog_slug"), + ) + op.create_index("ix_service_catalog_slug", "service_catalog", ["slug"]) + op.create_index("ix_service_catalog_hosting_type", "service_catalog", ["hosting_type"]) + op.create_index("ix_service_catalog_development_type", "service_catalog", ["development_type"]) + + op.create_table( + "service_third_party", + sa.Column("service_id", UUID(as_uuid=True), + sa.ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True), + sa.Column("pricing_model", sa.String(length=20), server_default="unknown", nullable=False), + sa.Column("upstream_packages", JSON(), nullable=True), + sa.Column("upstream_contacts", JSON(), nullable=True), + sa.Column("source_url", sa.Text(), nullable=True), + sa.Column("support_url", sa.Text(), nullable=True), + sa.Column("license", sa.String(length=100), nullable=True), + ) + + op.create_table( + "service_first_party", + sa.Column("service_id", UUID(as_uuid=True), + sa.ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True), + sa.Column("repo_id", UUID(as_uuid=True), + sa.ForeignKey("managed_repos.id", ondelete="SET NULL"), nullable=True), + sa.Column("owning_domain", sa.String(length=100), nullable=True), + ) + op.create_index("ix_service_first_party_repo_id", "service_first_party", ["repo_id"]) + + op.create_table( + "service_cloud", + sa.Column("service_id", UUID(as_uuid=True), + sa.ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True), + sa.Column("gdpr_maturity", sa.String(length=20), server_default="unknown", nullable=False), + sa.Column("gdpr_notes", sa.Text(), nullable=True), + sa.Column("dpa_available", sa.Boolean(), server_default=sa.text("false"), nullable=False), + sa.Column("tos_url", sa.Text(), nullable=True), + sa.Column("privacy_policy_url", sa.Text(), nullable=True), + sa.Column("data_processing_regions", JSON(), nullable=True), + sa.Column("data_retention_notes", sa.Text(), nullable=True), + ) + op.create_index("ix_service_cloud_gdpr_maturity", "service_cloud", ["gdpr_maturity"]) + + op.create_table( + "service_self_hosted", + sa.Column("service_id", UUID(as_uuid=True), + sa.ForeignKey("service_catalog.id", ondelete="CASCADE"), primary_key=True), + sa.Column("helix_instance", sa.String(length=100), nullable=True), + sa.Column("host_node", sa.String(length=100), nullable=True), + sa.Column("deployment_ref", sa.Text(), nullable=True), + sa.Column("runbook_ref", sa.Text(), nullable=True), + sa.Column("upstream_oss_project", sa.String(length=200), nullable=True), + ) + + # ── Data migration: tpsc_catalog → service_catalog (cloud_hosted, third_party) + op.execute( + """ + INSERT INTO service_catalog + (id, slug, name, owner_or_provider, category, website_url, status, + hosting_type, development_type, maturity_level, created_at, updated_at) + SELECT id, slug, name, provider, category, website_url, status, + 'cloud_hosted', 'third_party', NULL, created_at, updated_at + FROM tpsc_catalog + """ + ) + op.execute( + """ + INSERT INTO service_third_party (service_id, pricing_model) + SELECT id, pricing_model FROM tpsc_catalog + """ + ) + op.execute( + """ + INSERT INTO service_cloud + (service_id, gdpr_maturity, gdpr_notes, dpa_available, tos_url, + privacy_policy_url, data_processing_regions, data_retention_notes) + SELECT id, gdpr_maturity, gdpr_notes, dpa_available, tos_url, + privacy_policy_url, data_processing_regions, data_retention_notes + FROM tpsc_catalog + """ + ) + + +def downgrade() -> None: + op.drop_index("ix_service_cloud_gdpr_maturity", table_name="service_cloud") + op.drop_table("service_self_hosted") + op.drop_table("service_cloud") + op.drop_index("ix_service_first_party_repo_id", table_name="service_first_party") + op.drop_table("service_first_party") + op.drop_table("service_third_party") + op.drop_index("ix_service_catalog_development_type", table_name="service_catalog") + op.drop_index("ix_service_catalog_hosting_type", table_name="service_catalog") + op.drop_index("ix_service_catalog_slug", table_name="service_catalog") + op.drop_table("service_catalog")