diff --git a/docs/OPERATOR.md b/docs/OPERATOR.md index 55a0f68..3a9986c 100644 --- a/docs/OPERATOR.md +++ b/docs/OPERATOR.md @@ -65,6 +65,7 @@ All settings are prefixed with ``ARTIFACTSTORE_`` and read by | `ARTIFACTSTORE_S3_KEY_PREFIX` | empty | Optional object-key prefix before `/`. | | `ARTIFACTSTORE_S3_ACCESS_KEY_REF` | empty | Access key reference, `env:NAME` or `file:/mounted/path`. | | `ARTIFACTSTORE_S3_SECRET_KEY_REF` | empty | Secret key reference, `env:NAME` or `file:/mounted/path`. | +| `ARTIFACTSTORE_S3_SESSION_TOKEN_REF` | empty | Optional STS session token reference for temporary credentials, `env:NAME` or `file:/mounted/path`. When any credential ref is `file:`-based, all refs are re-resolved per client, so a sidecar/controller can rotate the three values atomically without a restart. | | `ARTIFACTSTORE_S3_STORAGE_CLASS` | empty | Optional storage class sent on writes. | | `ARTIFACTSTORE_S3_SSE` | empty | Optional server-side encryption value, e.g. `AES256`. | | `ARTIFACTSTORE_S3_MULTIPART_THRESHOLD_BYTES` | `67108864` | Multipart threshold for the S3 backend. | diff --git a/scripts/minio_local_smoke.sh b/scripts/minio_local_smoke.sh index b8268db..3bad744 100755 --- a/scripts/minio_local_smoke.sh +++ b/scripts/minio_local_smoke.sh @@ -55,4 +55,59 @@ ARTIFACTSTORE_MINIO_SECRET_KEY="$SECRET_KEY" \ ARTIFACTSTORE_MINIO_BUCKET="$BUCKET" \ make test-minio -echo "[minio-smoke] PASS — live MinIO round-trip/range/multipart compatibility verified" +echo "[minio-smoke] static-credential compatibility PASS" + +# ── STS leg (D7.4): temporary credentials via MinIO AssumeRole ──────────────── +# Root credentials cannot call AssumeRole, so mint a scoped user first. +STS_USER="sts-$(openssl rand -hex 6)" +STS_USER_SECRET="$(openssl rand -hex 24)" +docker exec -e STS_USER="$STS_USER" -e STS_USER_SECRET="$STS_USER_SECRET" "$CONTAINER" sh -c \ + 'mc admin user add local "$STS_USER" "$STS_USER_SECRET" >/dev/null && mc admin policy attach local readwrite --user "$STS_USER" >/dev/null' +echo "[minio-smoke] scoped user created; requesting temporary credentials via STS AssumeRole" + +STS_JSON="$( + STS_ENDPOINT="http://127.0.0.1:${MINIO_PORT}" \ + STS_USER="$STS_USER" STS_USER_SECRET="$STS_USER_SECRET" \ + uv run --all-extras python - <<'PY' +import json +import os + +import boto3 + +sts = boto3.client( + "sts", + endpoint_url=os.environ["STS_ENDPOINT"], + aws_access_key_id=os.environ["STS_USER"], + aws_secret_access_key=os.environ["STS_USER_SECRET"], + region_name="us-east-1", +) +creds = sts.assume_role( + RoleArn="arn:minio:iam:::role/dummy", + RoleSessionName="artifactstore-d74-smoke", + DurationSeconds=900, +)["Credentials"] +print( + json.dumps( + { + "AccessKeyId": creds["AccessKeyId"], + "SecretAccessKey": creds["SecretAccessKey"], + "SessionToken": creds["SessionToken"], + } + ) +) +PY +)" + +TEMP_ACCESS_KEY="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["AccessKeyId"])' "$STS_JSON")" +TEMP_SECRET_KEY="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["SecretAccessKey"])' "$STS_JSON")" +TEMP_SESSION_TOKEN="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["SessionToken"])' "$STS_JSON")" + +ARTIFACTSTORE_MINIO_ENDPOINT_URL="http://127.0.0.1:${MINIO_PORT}" \ +ARTIFACTSTORE_MINIO_ACCESS_KEY="$TEMP_ACCESS_KEY" \ +ARTIFACTSTORE_MINIO_SECRET_KEY="$TEMP_SECRET_KEY" \ +ARTIFACTSTORE_MINIO_SESSION_TOKEN="$TEMP_SESSION_TOKEN" \ +ARTIFACTSTORE_MINIO_BUCKET="$BUCKET" \ + make test-minio + +echo "[minio-smoke] temporary-credential (STS session token) compatibility PASS" +echo "[minio-smoke] PASS — live MinIO static + STS round-trip/range/multipart compatibility verified" diff --git a/src/artifactstore/app.py b/src/artifactstore/app.py index 0ddbcfc..486e19a 100644 --- a/src/artifactstore/app.py +++ b/src/artifactstore/app.py @@ -46,16 +46,24 @@ def _build_backends(settings: Settings) -> dict[str, StorageBackend]: if "local" in configured: backends["local"] = LocalBackend(settings.storage_local_root, backend_id="local") if "s3" in configured: - access_key = ( - resolve_secret_ref(settings.s3_access_key_ref) - if settings.s3_access_key_ref - else None - ) - secret_key = ( - resolve_secret_ref(settings.s3_secret_key_ref) - if settings.s3_secret_key_ref - else None + refs = ( + settings.s3_access_key_ref, + settings.s3_secret_key_ref, + settings.s3_session_token_ref, ) + + def _resolve_credentials() -> tuple[str | None, str | None, str | None]: + access_ref, secret_ref, token_ref = refs + return ( + resolve_secret_ref(access_ref) if access_ref else None, + resolve_secret_ref(secret_ref) if secret_ref else None, + resolve_secret_ref(token_ref) if token_ref else None, + ) + + access_key, secret_key, session_token = _resolve_credentials() + # file: refs are re-read per client so an STS sidecar/controller can + # rotate all three values atomically without a process restart. + refreshable = any(ref.startswith("file:") for ref in refs if ref) backends["s3"] = S3Backend( S3BackendConfig( endpoint_url=settings.s3_endpoint_url, @@ -64,11 +72,13 @@ def _build_backends(settings: Settings) -> dict[str, StorageBackend]: key_prefix=settings.s3_key_prefix, access_key_id=access_key, secret_access_key=secret_key, + session_token=session_token, storage_class=settings.s3_storage_class or None, sse=settings.s3_sse or None, multipart_threshold_bytes=settings.s3_multipart_threshold_bytes, multipart_chunk_bytes=settings.s3_multipart_chunk_bytes, - ) + ), + credentials_provider=_resolve_credentials if refreshable else None, ) unknown = set(configured) - set(backends) if unknown: diff --git a/src/artifactstore/config.py b/src/artifactstore/config.py index 43a72ee..3290b61 100644 --- a/src/artifactstore/config.py +++ b/src/artifactstore/config.py @@ -62,6 +62,7 @@ class Settings(BaseSettings): s3_key_prefix: str = "" s3_access_key_ref: str = "" s3_secret_key_ref: str = "" + s3_session_token_ref: str = "" s3_storage_class: str = "" s3_sse: str = "" s3_multipart_threshold_bytes: int = 64 * 1024 * 1024 diff --git a/src/artifactstore/storage/backends/s3.py b/src/artifactstore/storage/backends/s3.py index 87bf3d7..0138164 100644 --- a/src/artifactstore/storage/backends/s3.py +++ b/src/artifactstore/storage/backends/s3.py @@ -29,6 +29,7 @@ class S3BackendConfig: key_prefix: str = "" access_key_id: str | None = None secret_access_key: str | None = None + session_token: str | None = None storage_class: str | None = None sse: str | None = None multipart_threshold_bytes: int = 64 * 1024 * 1024 @@ -37,6 +38,11 @@ class S3BackendConfig: ClientFactory = Callable[[], AbstractAsyncContextManager[Any]] +# Returns (access_key_id, secret_access_key, session_token) for each new +# client, so STS-vended credentials rotated by a sidecar/controller (e.g. +# re-written mounted files) are picked up without a process restart. +CredentialsProvider = Callable[[], tuple[str | None, str | None, str | None]] + class S3Backend: """Storage SPI implementation over an S3-compatible object store.""" @@ -47,6 +53,7 @@ class S3Backend: *, backend_id: str = "s3", client_factory: ClientFactory | None = None, + credentials_provider: CredentialsProvider | None = None, chunk_size: int = _DEFAULT_CHUNK_SIZE, ) -> None: if not config.bucket: @@ -54,6 +61,7 @@ class S3Backend: self._config = config self._backend_id = backend_id self._client_factory = client_factory + self._credentials_provider = credentials_provider self._chunk_size = chunk_size @property @@ -69,9 +77,16 @@ class S3Backend: raise RuntimeError( "S3Backend requires the 'aioboto3' package; install artifactstore[s3]" ) from exc + if self._credentials_provider is not None: + access_key_id, secret_access_key, session_token = self._credentials_provider() + else: + access_key_id = self._config.access_key_id + secret_access_key = self._config.secret_access_key + session_token = self._config.session_token session = aioboto3.Session( - aws_access_key_id=self._config.access_key_id, - aws_secret_access_key=self._config.secret_access_key, + aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key, + aws_session_token=session_token, region_name=self._config.region, ) return cast( diff --git a/tests/integration/test_storage_s3_minio.py b/tests/integration/test_storage_s3_minio.py index 5a3521a..2ff5f2b 100644 --- a/tests/integration/test_storage_s3_minio.py +++ b/tests/integration/test_storage_s3_minio.py @@ -51,6 +51,7 @@ def minio_backend() -> S3Backend: key_prefix=environ.get("ARTIFACTSTORE_MINIO_KEY_PREFIX", f"compat/{uuid4()}"), access_key_id=environ["ARTIFACTSTORE_MINIO_ACCESS_KEY"], secret_access_key=environ["ARTIFACTSTORE_MINIO_SECRET_KEY"], + session_token=environ.get("ARTIFACTSTORE_MINIO_SESSION_TOKEN") or None, storage_class=environ.get("ARTIFACTSTORE_MINIO_STORAGE_CLASS") or None, sse=environ.get("ARTIFACTSTORE_MINIO_SSE") or None, multipart_threshold_bytes=5 * _MIB, diff --git a/tests/unit/test_storage_s3.py b/tests/unit/test_storage_s3.py index dc23543..be5917f 100644 --- a/tests/unit/test_storage_s3.py +++ b/tests/unit/test_storage_s3.py @@ -2,7 +2,9 @@ from __future__ import annotations +import sys from collections.abc import AsyncIterator +from types import SimpleNamespace from typing import Any import pytest @@ -194,3 +196,55 @@ async def test_health_uses_head_bucket(backend: S3Backend) -> None: status = await backend.health() assert status.healthy is True assert status.backend_id == "s3" + + +def test_client_passes_session_token(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + + class FakeSession: + def __init__(self, **kwargs: object) -> None: + captured.update(kwargs) + + def client(self, *args: object, **kwargs: object) -> object: + return object() + + monkeypatch.setitem(sys.modules, "aioboto3", SimpleNamespace(Session=FakeSession)) + backend_with_token = S3Backend( + S3BackendConfig( + endpoint_url="http://minio.local:9000", + region="us-east-1", + bucket="bucket", + access_key_id="AKIA-temporary", + secret_access_key="temp-secret", + session_token="temp-session-token", + ) + ) + backend_with_token._client() + assert captured["aws_session_token"] == "temp-session-token" + + +def test_credentials_provider_re_resolves_per_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen_tokens: list[object] = [] + + class FakeSession: + def __init__(self, **kwargs: object) -> None: + seen_tokens.append(kwargs.get("aws_session_token")) + + def client(self, *args: object, **kwargs: object) -> object: + return object() + + monkeypatch.setitem(sys.modules, "aioboto3", SimpleNamespace(Session=FakeSession)) + rotation = iter(["token-1", "token-2"]) + backend_rotating = S3Backend( + S3BackendConfig( + endpoint_url="http://minio.local:9000", + region="us-east-1", + bucket="bucket", + ), + credentials_provider=lambda: ("key", "secret", next(rotation)), + ) + backend_rotating._client() + backend_rotating._client() + assert seen_tokens == ["token-1", "token-2"] diff --git a/workplans/ARTIFACT-STORE-WP-0007-minio-maxio-sts-vending.md b/workplans/ARTIFACT-STORE-WP-0007-minio-maxio-sts-vending.md index 53ef6ee..ee3308f 100644 --- a/workplans/ARTIFACT-STORE-WP-0007-minio-maxio-sts-vending.md +++ b/workplans/ARTIFACT-STORE-WP-0007-minio-maxio-sts-vending.md @@ -169,7 +169,7 @@ consumed until that lands. ```task id: ARTIFACT-STORE-WP-0007-T004 -status: todo +status: done priority: medium state_hub_task_id: "9b80057a-d86e-4f14-9d14-928ee29f970d" ``` @@ -184,6 +184,25 @@ Acceptance: - Verify that `artifactstore storage verify --backend s3` can run with temporary credentials. +Completed 2026-07-02: + +- Decision: in-process refresh uses per-client re-resolution of `file:` refs + (sidecar/controller rewrites the mounted files atomically); no long-lived + credential state is cached, and values never enter request bodies, events, + or config dumps. +- Config shape: `S3BackendConfig.session_token` + + `ARTIFACTSTORE_S3_SESSION_TOKEN_REF` (env:/file: ref like the existing key + refs); `S3Backend` accepts an optional `credentials_provider` returning + (access, secret, token) per client. +- Live verification against a local MinIO: the smoke's new STS leg mints + temporary credentials via `AssumeRole` for a scoped non-root user and + passes round-trip/range/multipart with the session token + (`make test-minio-local`), and the CLI ran `migrate`/`health`/`storage + verify --backend s3` with STS credentials delivered through `file:` refs — + backend health `ok` proves a live authenticated `head_bucket`. +- `make test` 112 passed / 2 skipped; targeted Ruff clean. Unit tests cover + session-token pass-through and per-client provider re-resolution. + ## D7.5 - Follow-Up Workstream Routing ```task