Implement app deployment improvements
This commit is contained in:
22
.gitea/workflows/manifest-server-dry-run.yaml
Normal file
22
.gitea/workflows/manifest-server-dry-run.yaml
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
name: Manifest server dry-run
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- "charts/**"
|
||||||
|
- "helm/**"
|
||||||
|
- "manifests/**"
|
||||||
|
- "tools/k8s-server-dry-run.sh"
|
||||||
|
- "Makefile"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
dry-run:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check out repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Server-side dry-run manifests
|
||||||
|
env:
|
||||||
|
DRY_RUN_CREATE_NAMESPACES: "true"
|
||||||
|
run: make k8s-server-dry-run
|
||||||
52
Makefile
52
Makefile
@@ -14,6 +14,27 @@ VERGABE_CHART ?= charts/vergabe-teilnahme
|
|||||||
VERGABE_VALUES ?= helm/vergabe-teilnahme-values.yaml
|
VERGABE_VALUES ?= helm/vergabe-teilnahme-values.yaml
|
||||||
VERGABE_INGRESS ?= manifests/vergabe-teilnahme-ingress.yaml
|
VERGABE_INGRESS ?= manifests/vergabe-teilnahme-ingress.yaml
|
||||||
|
|
||||||
|
VERGABE_DB_SECRET ?= vergabe-app-credentials
|
||||||
|
VERGABE_ENV_SECRET ?= vergabe-teilnahme-env
|
||||||
|
VERGABE_DB_USER ?= vergabe
|
||||||
|
VERGABE_DB_HOST ?= apps-pg-rw.databases
|
||||||
|
VERGABE_DB_PORT ?= 5432
|
||||||
|
VERGABE_DB_NAME ?= vergabe_db
|
||||||
|
|
||||||
|
SOPS_SENTINEL ?= $(GITEA_VALUES)
|
||||||
|
DRY_RUN_CREATE_NAMESPACES ?= false
|
||||||
|
|
||||||
|
##@ Operator checks
|
||||||
|
|
||||||
|
check-tools: ## Check required operator tools and warn about optional diagnostics
|
||||||
|
tools/check-tools.sh
|
||||||
|
|
||||||
|
check-sops: ## Verify the local SOPS age key can decrypt the configured sentinel
|
||||||
|
SOPS_SENTINEL=$(SOPS_SENTINEL) tools/check-sops.sh
|
||||||
|
|
||||||
|
k8s-server-dry-run: ## Server-side dry-run rendered Helm and committed manifests
|
||||||
|
DRY_RUN_CREATE_NAMESPACES=$(DRY_RUN_CREATE_NAMESPACES) tools/k8s-server-dry-run.sh
|
||||||
|
|
||||||
##@ Gitea
|
##@ Gitea
|
||||||
|
|
||||||
gitea-deploy: ## Deploy / upgrade Gitea (S5 workload)
|
gitea-deploy: ## Deploy / upgrade Gitea (S5 workload)
|
||||||
@@ -29,7 +50,22 @@ gitea-status: ## Check Gitea health
|
|||||||
kubectl get pods -n $(GITEA_NAMESPACE) -l app.kubernetes.io/instance=$(GITEA_RELEASE)
|
kubectl get pods -n $(GITEA_NAMESPACE) -l app.kubernetes.io/instance=$(GITEA_RELEASE)
|
||||||
kubectl get svc -n $(GITEA_NAMESPACE) $(GITEA_RELEASE)
|
kubectl get svc -n $(GITEA_NAMESPACE) $(GITEA_RELEASE)
|
||||||
kubectl get ingress -n $(GITEA_NAMESPACE) $(GITEA_RELEASE) --ignore-not-found
|
kubectl get ingress -n $(GITEA_NAMESPACE) $(GITEA_RELEASE) --ignore-not-found
|
||||||
kubectl cnpg status gitea-db -n databases
|
@if kubectl cnpg status gitea-db -n databases >/dev/null 2>&1; then \
|
||||||
|
kubectl cnpg status gitea-db -n databases; \
|
||||||
|
else \
|
||||||
|
echo "kubectl cnpg plugin not available; falling back to cnpg resources"; \
|
||||||
|
kubectl get cluster gitea-db -n databases; \
|
||||||
|
kubectl get pods -n databases -l cnpg.io/cluster=gitea-db; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
apps-pg-status: ## Check the shared apps-pg cnpg cluster
|
||||||
|
@if kubectl cnpg status apps-pg -n databases >/dev/null 2>&1; then \
|
||||||
|
kubectl cnpg status apps-pg -n databases; \
|
||||||
|
else \
|
||||||
|
echo "kubectl cnpg plugin not available; falling back to cnpg resources"; \
|
||||||
|
kubectl get cluster apps-pg -n databases; \
|
||||||
|
kubectl get pods -n databases -l cnpg.io/cluster=apps-pg; \
|
||||||
|
fi
|
||||||
|
|
||||||
##@ Vergabe Teilnahme
|
##@ Vergabe Teilnahme
|
||||||
|
|
||||||
@@ -61,11 +97,21 @@ vergabe-superuser: ## Open an interactive shell for createsuperuser
|
|||||||
vergabe-logs: ## Tail vergabe-teilnahme app logs
|
vergabe-logs: ## Tail vergabe-teilnahme app logs
|
||||||
kubectl logs -n $(VERGABE_NAMESPACE) -l app.kubernetes.io/instance=$(VERGABE_RELEASE) -f --tail=50
|
kubectl logs -n $(VERGABE_NAMESPACE) -l app.kubernetes.io/instance=$(VERGABE_RELEASE) -f --tail=50
|
||||||
|
|
||||||
|
vergabe-db-url-secret: ## Rebuild DATABASE_URL with a URL-encoded cnpg password
|
||||||
|
APP_NAMESPACE=$(VERGABE_NAMESPACE) \
|
||||||
|
APP_ENV_SECRET=$(VERGABE_ENV_SECRET) \
|
||||||
|
APP_DB_SECRET=$(VERGABE_DB_SECRET) \
|
||||||
|
APP_DB_USER=$(VERGABE_DB_USER) \
|
||||||
|
APP_DB_HOST=$(VERGABE_DB_HOST) \
|
||||||
|
APP_DB_PORT=$(VERGABE_DB_PORT) \
|
||||||
|
APP_DB_NAME=$(VERGABE_DB_NAME) \
|
||||||
|
tools/build-database-url-secret.sh
|
||||||
|
|
||||||
##@ Help
|
##@ Help
|
||||||
|
|
||||||
help: ## Show this help
|
help: ## Show this help
|
||||||
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} \
|
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} \
|
||||||
/^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } \
|
/^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } \
|
||||||
/^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) }' $(MAKEFILE_LIST)
|
/^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) }' $(MAKEFILE_LIST)
|
||||||
|
|
||||||
.PHONY: gitea-deploy gitea-ingress-deploy gitea-status vergabe-dry-run vergabe-deploy vergabe-ingress-deploy vergabe-status vergabe-migrate vergabe-seed vergabe-superuser vergabe-logs help
|
.PHONY: check-tools check-sops k8s-server-dry-run gitea-deploy gitea-ingress-deploy gitea-status apps-pg-status vergabe-dry-run vergabe-deploy vergabe-ingress-deploy vergabe-status vergabe-migrate vergabe-seed vergabe-superuser vergabe-logs vergabe-db-url-secret help
|
||||||
|
|||||||
54
docs/django-on-railiance.md
Normal file
54
docs/django-on-railiance.md
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
# Django on Railiance
|
||||||
|
|
||||||
|
This is the short recipe for Django workloads running as S5 apps on the
|
||||||
|
Railiance cluster.
|
||||||
|
|
||||||
|
## Probe Host Header
|
||||||
|
|
||||||
|
Production Django usually runs with `DEBUG=False` and a narrow
|
||||||
|
`ALLOWED_HOSTS`. Kubelet HTTP probes do not use the public app host by
|
||||||
|
default; they send the pod IP as the `Host` header. Django rejects that
|
||||||
|
before the request reaches the health view, so kubelet sees `HTTP 400`
|
||||||
|
and restarts an otherwise healthy pod.
|
||||||
|
|
||||||
|
Set the probe `Host` header to a value that is also present in
|
||||||
|
`ALLOWED_HOSTS`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
env:
|
||||||
|
ALLOWED_HOSTS: app.example.org,localhost
|
||||||
|
|
||||||
|
probes:
|
||||||
|
enabled: true
|
||||||
|
path: /health/
|
||||||
|
port: 8000
|
||||||
|
hostHeader: app.example.org
|
||||||
|
```
|
||||||
|
|
||||||
|
The `charts/vergabe-teilnahme` chart already renders this as:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
httpGet:
|
||||||
|
path: /health/
|
||||||
|
port: 8000
|
||||||
|
httpHeaders:
|
||||||
|
- name: Host
|
||||||
|
value: app.example.org
|
||||||
|
```
|
||||||
|
|
||||||
|
When changing the public hostname or `ALLOWED_HOSTS`, change
|
||||||
|
`probes.hostHeader` in the same review.
|
||||||
|
|
||||||
|
## Database URL Secrets
|
||||||
|
|
||||||
|
CNPG-generated role passwords may contain URL-reserved characters such
|
||||||
|
as `=`, `+`, and `/`. If the app consumes `DATABASE_URL`, URL-encode the
|
||||||
|
password when building the env Secret:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make vergabe-db-url-secret
|
||||||
|
```
|
||||||
|
|
||||||
|
For new charts, prefer either this helper or separate PostgreSQL env
|
||||||
|
vars (`POSTGRES_HOST`, `POSTGRES_USER`, `POSTGRES_PASSWORD`,
|
||||||
|
`POSTGRES_DB`) so there is no URL parser in the path.
|
||||||
@@ -9,8 +9,8 @@ HTTPS.
|
|||||||
Registry-specific Gitea settings are carried in
|
Registry-specific Gitea settings are carried in
|
||||||
`helm/gitea-registry-values.yaml`, a non-secret overlay applied after the SOPS
|
`helm/gitea-registry-values.yaml`, a non-secret overlay applied after the SOPS
|
||||||
values file by `make gitea-deploy`. It explicitly enables packages, permits
|
values file by `make gitea-deploy`. It explicitly enables packages, permits
|
||||||
container uploads without an app-level size cap, clears globally disabled repo
|
container and PyPI uploads without an app-level size cap, clears globally
|
||||||
units, and moves `ROOT_URL` to the HTTPS host.
|
disabled repo units, and moves `ROOT_URL` to the HTTPS host.
|
||||||
|
|
||||||
Image names should use the Gitea owner and package path:
|
Image names should use the Gitea owner and package path:
|
||||||
|
|
||||||
@@ -60,6 +60,12 @@ kubectl create secret docker-registry gitea-registry \
|
|||||||
|
|
||||||
Reference it from workloads as `imagePullSecrets: [{name: gitea-registry}]`.
|
Reference it from workloads as `imagePullSecrets: [{name: gitea-registry}]`.
|
||||||
|
|
||||||
|
## Python Packages
|
||||||
|
|
||||||
|
The same Gitea package service is used for Python wheels. See
|
||||||
|
`docs/gitea-package-registry.md` for the publish/install recipe and the
|
||||||
|
`issue-core` migration notes from `RAILIANCE-WP-0004 I03`.
|
||||||
|
|
||||||
## Current Storage Notes
|
## Current Storage Notes
|
||||||
|
|
||||||
The live Gitea pod mounts `gitea-shared-storage` at `/data`; package blobs land
|
The live Gitea pod mounts `gitea-shared-storage` at `/data`; package blobs land
|
||||||
|
|||||||
48
docs/gitea-package-registry.md
Normal file
48
docs/gitea-package-registry.md
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# Gitea Package Registry
|
||||||
|
|
||||||
|
Gitea package support is enabled by `helm/gitea-registry-values.yaml`.
|
||||||
|
That overlay is applied after the encrypted base values by
|
||||||
|
`make gitea-deploy` and enables both container packages and Python
|
||||||
|
packages.
|
||||||
|
|
||||||
|
## Python Packages
|
||||||
|
|
||||||
|
Publish Python wheels to the organization package endpoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m build
|
||||||
|
TWINE_USERNAME=<gitea-user> \
|
||||||
|
TWINE_PASSWORD=<package-token> \
|
||||||
|
python -m twine upload \
|
||||||
|
--repository-url https://gitea.coulomb.social/api/packages/coulomb/pypi \
|
||||||
|
dist/*
|
||||||
|
```
|
||||||
|
|
||||||
|
Install from the simple index:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install \
|
||||||
|
--extra-index-url https://<gitea-user>:<package-token>@gitea.coulomb.social/api/packages/coulomb/pypi/simple/ \
|
||||||
|
issue-core
|
||||||
|
```
|
||||||
|
|
||||||
|
For CI, store the token as a secret and inject it into the package index
|
||||||
|
URL at build time. Do not commit tokenized index URLs.
|
||||||
|
|
||||||
|
## issue-core Migration
|
||||||
|
|
||||||
|
The portable deployment path for `vergabe-teilnahme` is:
|
||||||
|
|
||||||
|
1. Release `issue-core` from its source repo as a wheel, for example
|
||||||
|
`0.2.0`.
|
||||||
|
2. Publish the wheel to the Gitea Python package registry.
|
||||||
|
3. Change `vergabe-teilnahme/pyproject.toml` from the local path
|
||||||
|
dependency to `issue-core>=0.2,<0.3`.
|
||||||
|
4. Remove the Docker BuildKit `--build-context issue-core=...`
|
||||||
|
requirement from image-build instructions.
|
||||||
|
5. Build the image on a clean runner that has no sibling `issue-core`
|
||||||
|
checkout.
|
||||||
|
|
||||||
|
Only the registry enablement lives in `railiance-apps`; the package
|
||||||
|
release and application dependency change belong to the `issue-core` and
|
||||||
|
`vergabe-teilnahme` repos.
|
||||||
43
docs/operator-recipes.md
Normal file
43
docs/operator-recipes.md
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
# Operator Recipes
|
||||||
|
|
||||||
|
## Service-IP Smoke Checks
|
||||||
|
|
||||||
|
Avoid one-shot `kubectl run --rm -i` probes for service connectivity.
|
||||||
|
The container can exit before the connection result is reliable, which
|
||||||
|
creates false negatives during rollout debugging.
|
||||||
|
|
||||||
|
Use a persistent pod, wait for readiness, then exec the probe:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NAMESPACE=vergabe-teilnahme \
|
||||||
|
tools/smoke-service.sh http://vergabe-teilnahme.vergabe-teilnahme.svc/health/
|
||||||
|
```
|
||||||
|
|
||||||
|
Reuse the same pod for a debugging session:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NAMESPACE=vergabe-teilnahme POD_NAME=service-smoke \
|
||||||
|
tools/smoke-service.sh http://vergabe-teilnahme.vergabe-teilnahme.svc/health/
|
||||||
|
```
|
||||||
|
|
||||||
|
Clean it up when finished:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl delete pod service-smoke -n vergabe-teilnahme
|
||||||
|
```
|
||||||
|
|
||||||
|
Or set `CLEANUP=true` for a single checked run.
|
||||||
|
|
||||||
|
## Manifest Server Dry-Run
|
||||||
|
|
||||||
|
Schema drift in live CRDs is caught by server-side dry-run, not by Helm
|
||||||
|
rendering alone:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make k8s-server-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
The command expects a representative Kubernetes API server with the same
|
||||||
|
CRDs as the Railiance cluster. CI should run it against a disposable kind
|
||||||
|
cluster seeded with CNPG, cert-manager, Traefik, and any other CRDs used
|
||||||
|
by changed manifests.
|
||||||
62
docs/operator-setup.md
Normal file
62
docs/operator-setup.md
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
# Operator Setup
|
||||||
|
|
||||||
|
Run these checks before deploying or rotating any S5 workload:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make check-tools
|
||||||
|
make check-sops
|
||||||
|
```
|
||||||
|
|
||||||
|
## Required Tools
|
||||||
|
|
||||||
|
- `kubectl`
|
||||||
|
- `helm`
|
||||||
|
- `sops`
|
||||||
|
- `python3`
|
||||||
|
|
||||||
|
Install the CNPG plugin for better database diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl krew install cnpg
|
||||||
|
```
|
||||||
|
|
||||||
|
`make check-tools` fails when required tools are missing and warns when
|
||||||
|
`kubectl cnpg` is unavailable. The Makefile status targets fall back to
|
||||||
|
plain Kubernetes resources, but the plugin output is the preferred view
|
||||||
|
for primary/replica health and backup state.
|
||||||
|
|
||||||
|
## SOPS Age Key Bootstrap
|
||||||
|
|
||||||
|
SOPS-encrypted values in this repo expect an age identity at:
|
||||||
|
|
||||||
|
```text
|
||||||
|
~/.config/sops/age/keys.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Bootstrap procedure:
|
||||||
|
|
||||||
|
1. Receive the operator age identity through an out-of-band channel.
|
||||||
|
2. Create the directory with owner-only permissions:
|
||||||
|
```bash
|
||||||
|
mkdir -p ~/.config/sops/age
|
||||||
|
chmod 700 ~/.config/sops ~/.config/sops/age
|
||||||
|
```
|
||||||
|
3. Write the identity to `~/.config/sops/age/keys.txt`.
|
||||||
|
4. Restrict the file:
|
||||||
|
```bash
|
||||||
|
chmod 600 ~/.config/sops/age/keys.txt
|
||||||
|
```
|
||||||
|
5. Verify decryption:
|
||||||
|
```bash
|
||||||
|
make check-sops
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not commit age identities, decrypted values, or copied SOPS plaintext
|
||||||
|
to this repo.
|
||||||
|
|
||||||
|
## Rotation
|
||||||
|
|
||||||
|
To rotate access, add the new recipient to the relevant SOPS files,
|
||||||
|
re-encrypt, verify with both old and new operators, then remove the old
|
||||||
|
recipient in a separate change. Keep at least one known-good recovery
|
||||||
|
operator key available during the transition.
|
||||||
@@ -36,13 +36,9 @@ K8s Secrets, not in committed values files.
|
|||||||
2. Mirror the new password into `vergabe-teilnahme/vergabe-app-credentials`.
|
2. Mirror the new password into `vergabe-teilnahme/vergabe-app-credentials`.
|
||||||
3. Rebuild `DATABASE_URL` in `vergabe-teilnahme-env`, **URL-encoding
|
3. Rebuild `DATABASE_URL` in `vergabe-teilnahme-env`, **URL-encoding
|
||||||
the password** (the base64 character set breaks the URL parser
|
the password** (the base64 character set breaks the URL parser
|
||||||
otherwise — see `RAILIANCE-WP-0004 I01`):
|
otherwise - see `RAILIANCE-WP-0004 I01`):
|
||||||
```bash
|
```bash
|
||||||
PW=$(kubectl get secret vergabe-app-credentials -n vergabe-teilnahme -o jsonpath='{.data.password}' | base64 -d)
|
make vergabe-db-url-secret
|
||||||
ENCODED=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "$PW")
|
|
||||||
kubectl patch secret vergabe-teilnahme-env -n vergabe-teilnahme \
|
|
||||||
--type=merge \
|
|
||||||
-p "{\"stringData\":{\"DATABASE_URL\":\"postgresql://vergabe:$ENCODED@apps-pg-rw.databases:5432/vergabe_db\"}}"
|
|
||||||
kubectl rollout restart deploy/vergabe-teilnahme -n vergabe-teilnahme
|
kubectl rollout restart deploy/vergabe-teilnahme -n vergabe-teilnahme
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -103,6 +99,7 @@ Most likely the probe's `Host` header doesn't match
|
|||||||
vergabe-teilnahme.whywhynot.de` precisely to avoid this — if you
|
vergabe-teilnahme.whywhynot.de` precisely to avoid this — if you
|
||||||
change `ALLOWED_HOSTS` in values, also update `probes.hostHeader`.
|
change `ALLOWED_HOSTS` in values, also update `probes.hostHeader`.
|
||||||
Symptom in `kubectl logs`: kube-probe requests returning HTTP 400.
|
Symptom in `kubectl logs`: kube-probe requests returning HTTP 400.
|
||||||
|
See `docs/django-on-railiance.md` for the reusable pattern.
|
||||||
|
|
||||||
### `dj-database-url` error: "The database name 'XYZ...' is longer than 63 characters"
|
### `dj-database-url` error: "The database name 'XYZ...' is longer than 63 characters"
|
||||||
|
|
||||||
@@ -156,4 +153,8 @@ configuration belongs to `railiance-platform`).
|
|||||||
- Improvements backlog: `workplans/railiance-apps-WP-0004-app-deployment-improvements.md`
|
- Improvements backlog: `workplans/railiance-apps-WP-0004-app-deployment-improvements.md`
|
||||||
- Shared DB cluster: `railiance-platform/docs/apps-pg.md`
|
- Shared DB cluster: `railiance-platform/docs/apps-pg.md`
|
||||||
- Container registry: `docs/gitea-container-registry.md`
|
- Container registry: `docs/gitea-container-registry.md`
|
||||||
|
- Python package registry: `docs/gitea-package-registry.md`
|
||||||
|
- Django deployment recipe: `docs/django-on-railiance.md`
|
||||||
|
- Operator setup: `docs/operator-setup.md`
|
||||||
|
- Operator recipes: `docs/operator-recipes.md`
|
||||||
- App source: https://gitea.coulomb.social/coulomb/vergabe-teilnahme
|
- App source: https://gitea.coulomb.social/coulomb/vergabe-teilnahme
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ gitea:
|
|||||||
packages:
|
packages:
|
||||||
ENABLED: true
|
ENABLED: true
|
||||||
LIMIT_SIZE_CONTAINER: -1
|
LIMIT_SIZE_CONTAINER: -1
|
||||||
|
LIMIT_SIZE_PYPI: -1
|
||||||
repository:
|
repository:
|
||||||
DISABLED_REPO_UNITS: ""
|
DISABLED_REPO_UNITS: ""
|
||||||
server:
|
server:
|
||||||
|
|||||||
74
tools/build-database-url-secret.sh
Executable file
74
tools/build-database-url-secret.sh
Executable file
@@ -0,0 +1,74 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Build or patch an application env Secret with a URL-encoded PostgreSQL DATABASE_URL.
|
||||||
|
|
||||||
|
Required environment:
|
||||||
|
APP_NAMESPACE Consumer namespace, for example vergabe-teilnahme
|
||||||
|
APP_ENV_SECRET Env Secret to create or patch, for example vergabe-teilnahme-env
|
||||||
|
APP_DB_SECRET Secret containing the raw cnpg role password
|
||||||
|
APP_DB_USER Database user
|
||||||
|
APP_DB_HOST Database host
|
||||||
|
APP_DB_NAME Database name
|
||||||
|
|
||||||
|
Optional environment:
|
||||||
|
APP_DB_PASSWORD_KEY Secret key containing the raw password (default: password)
|
||||||
|
APP_DB_PORT Database port (default: 5432)
|
||||||
|
APP_DB_SCHEME URL scheme (default: postgresql)
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
: "${APP_NAMESPACE:?Set APP_NAMESPACE}"
|
||||||
|
: "${APP_ENV_SECRET:?Set APP_ENV_SECRET}"
|
||||||
|
: "${APP_DB_SECRET:?Set APP_DB_SECRET}"
|
||||||
|
: "${APP_DB_USER:?Set APP_DB_USER}"
|
||||||
|
: "${APP_DB_HOST:?Set APP_DB_HOST}"
|
||||||
|
: "${APP_DB_NAME:?Set APP_DB_NAME}"
|
||||||
|
|
||||||
|
APP_DB_PASSWORD_KEY="${APP_DB_PASSWORD_KEY:-password}"
|
||||||
|
APP_DB_PORT="${APP_DB_PORT:-5432}"
|
||||||
|
APP_DB_SCHEME="${APP_DB_SCHEME:-postgresql}"
|
||||||
|
|
||||||
|
for cmd in kubectl base64 python3; do
|
||||||
|
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: missing required command: $cmd" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
raw_password="$(
|
||||||
|
kubectl get secret "$APP_DB_SECRET" \
|
||||||
|
-n "$APP_NAMESPACE" \
|
||||||
|
-o "jsonpath={.data.${APP_DB_PASSWORD_KEY}}" | base64 -d
|
||||||
|
)"
|
||||||
|
|
||||||
|
if [[ -z "$raw_password" ]]; then
|
||||||
|
echo "ERROR: secret $APP_NAMESPACE/$APP_DB_SECRET did not contain key $APP_DB_PASSWORD_KEY" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
encoded_password="$(
|
||||||
|
RAW_PASSWORD="$raw_password" python3 -c 'import os, urllib.parse; print(urllib.parse.quote(os.environ["RAW_PASSWORD"], safe=""))'
|
||||||
|
)"
|
||||||
|
database_url="${APP_DB_SCHEME}://${APP_DB_USER}:${encoded_password}@${APP_DB_HOST}:${APP_DB_PORT}/${APP_DB_NAME}"
|
||||||
|
|
||||||
|
if kubectl get secret "$APP_ENV_SECRET" -n "$APP_NAMESPACE" >/dev/null 2>&1; then
|
||||||
|
patch="$(
|
||||||
|
DATABASE_URL="$database_url" python3 -c 'import json, os; print(json.dumps({"stringData": {"DATABASE_URL": os.environ["DATABASE_URL"]}}))'
|
||||||
|
)"
|
||||||
|
kubectl patch secret "$APP_ENV_SECRET" -n "$APP_NAMESPACE" --type=merge -p "$patch"
|
||||||
|
else
|
||||||
|
kubectl create secret generic "$APP_ENV_SECRET" \
|
||||||
|
-n "$APP_NAMESPACE" \
|
||||||
|
--from-literal=DATABASE_URL="$database_url"
|
||||||
|
echo "WARN: created $APP_NAMESPACE/$APP_ENV_SECRET with DATABASE_URL only; add other required env keys separately" >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Updated DATABASE_URL in secret $APP_NAMESPACE/$APP_ENV_SECRET"
|
||||||
24
tools/check-sops.sh
Executable file
24
tools/check-sops.sh
Executable file
@@ -0,0 +1,24 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SOPS_SENTINEL="${SOPS_SENTINEL:-helm/gitea-values.sops.yaml}"
|
||||||
|
SOPS_AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}"
|
||||||
|
|
||||||
|
if ! command -v sops >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: sops is not installed" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -s "$SOPS_AGE_KEY_FILE" ]]; then
|
||||||
|
echo "ERROR: SOPS age key file is missing or empty: $SOPS_AGE_KEY_FILE" >&2
|
||||||
|
echo "Place the operator age identity there, or set SOPS_AGE_KEY_FILE to its path." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f "$SOPS_SENTINEL" ]]; then
|
||||||
|
echo "ERROR: sentinel file does not exist: $SOPS_SENTINEL" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
sops -d "$SOPS_SENTINEL" >/dev/null
|
||||||
|
echo "ok: decrypted $SOPS_SENTINEL with $SOPS_AGE_KEY_FILE"
|
||||||
32
tools/check-tools.sh
Executable file
32
tools/check-tools.sh
Executable file
@@ -0,0 +1,32 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
missing=0
|
||||||
|
|
||||||
|
check_required() {
|
||||||
|
local cmd="$1"
|
||||||
|
if command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
echo "ok: $cmd"
|
||||||
|
else
|
||||||
|
echo "ERROR: missing required tool: $cmd" >&2
|
||||||
|
missing=1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_required kubectl
|
||||||
|
check_required helm
|
||||||
|
check_required sops
|
||||||
|
check_required python3
|
||||||
|
|
||||||
|
if command -v kubectl >/dev/null 2>&1; then
|
||||||
|
if kubectl cnpg --help >/dev/null 2>&1; then
|
||||||
|
echo "ok: kubectl cnpg"
|
||||||
|
else
|
||||||
|
echo "WARN: kubectl cnpg plugin is missing; install with: kubectl krew install cnpg" >&2
|
||||||
|
if ! kubectl krew version >/dev/null 2>&1; then
|
||||||
|
echo "WARN: kubectl krew is missing; install krew before using kubectl krew install cnpg" >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit "$missing"
|
||||||
38
tools/k8s-server-dry-run.sh
Executable file
38
tools/k8s-server-dry-run.sh
Executable file
@@ -0,0 +1,38 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
|
||||||
|
VERGABE_RELEASE="${VERGABE_RELEASE:-vergabe-teilnahme}"
|
||||||
|
VERGABE_NAMESPACE="${VERGABE_NAMESPACE:-vergabe-teilnahme}"
|
||||||
|
VERGABE_CHART="${VERGABE_CHART:-charts/vergabe-teilnahme}"
|
||||||
|
VERGABE_VALUES="${VERGABE_VALUES:-helm/vergabe-teilnahme-values.yaml}"
|
||||||
|
DRY_RUN_CREATE_NAMESPACES="${DRY_RUN_CREATE_NAMESPACES:-false}"
|
||||||
|
|
||||||
|
for cmd in kubectl helm; do
|
||||||
|
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: missing required command: $cmd" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
kubectl api-resources >/dev/null
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN_CREATE_NAMESPACES" == "true" ]]; then
|
||||||
|
kubectl create namespace "$VERGABE_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
|
||||||
|
fi
|
||||||
|
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$tmpdir"' EXIT
|
||||||
|
|
||||||
|
helm template "$VERGABE_RELEASE" "$VERGABE_CHART" \
|
||||||
|
--namespace "$VERGABE_NAMESPACE" \
|
||||||
|
-f "$VERGABE_VALUES" \
|
||||||
|
> "$tmpdir/vergabe-teilnahme.yaml"
|
||||||
|
|
||||||
|
echo "server dry-run: committed manifests"
|
||||||
|
kubectl apply --dry-run=server -f manifests
|
||||||
|
|
||||||
|
echo "server dry-run: rendered $VERGABE_RELEASE chart"
|
||||||
|
kubectl apply --dry-run=server -n "$VERGABE_NAMESPACE" -f "$tmpdir/vergabe-teilnahme.yaml"
|
||||||
47
tools/smoke-service.sh
Executable file
47
tools/smoke-service.sh
Executable file
@@ -0,0 +1,47 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Run a service smoke test from a persistent pod, then exec curl inside it.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
NAMESPACE=<namespace> tools/smoke-service.sh http://service.namespace.svc/path
|
||||||
|
|
||||||
|
Optional environment:
|
||||||
|
POD_NAME Reusable smoke pod name (default: service-smoke)
|
||||||
|
SMOKE_IMAGE Image with curl installed (default: curlimages/curl:8.10.1)
|
||||||
|
CLEANUP Delete the smoke pod after the test (default: false)
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" || $# -ne 1 ]]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
target_url="$1"
|
||||||
|
NAMESPACE="${NAMESPACE:-default}"
|
||||||
|
POD_NAME="${POD_NAME:-service-smoke}"
|
||||||
|
SMOKE_IMAGE="${SMOKE_IMAGE:-curlimages/curl:8.10.1}"
|
||||||
|
CLEANUP="${CLEANUP:-false}"
|
||||||
|
|
||||||
|
if ! command -v kubectl >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: missing required command: kubectl" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! kubectl get pod "$POD_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||||
|
kubectl run "$POD_NAME" \
|
||||||
|
-n "$NAMESPACE" \
|
||||||
|
--image="$SMOKE_IMAGE" \
|
||||||
|
--restart=Never \
|
||||||
|
--command -- sleep 3600
|
||||||
|
fi
|
||||||
|
|
||||||
|
kubectl wait -n "$NAMESPACE" --for=condition=Ready "pod/$POD_NAME" --timeout=90s
|
||||||
|
kubectl exec -n "$NAMESPACE" "$POD_NAME" -- curl -fsS "$target_url"
|
||||||
|
|
||||||
|
if [[ "$CLEANUP" == "true" ]]; then
|
||||||
|
kubectl delete pod "$POD_NAME" -n "$NAMESPACE" --wait=false
|
||||||
|
fi
|
||||||
@@ -4,12 +4,12 @@ type: workplan
|
|||||||
title: "App deployment improvements (lessons from RAILIANCE-WP-0002)"
|
title: "App deployment improvements (lessons from RAILIANCE-WP-0002)"
|
||||||
domain: railiance
|
domain: railiance
|
||||||
repo: railiance-apps
|
repo: railiance-apps
|
||||||
status: backlog
|
status: active
|
||||||
owner: railiance
|
owner: railiance
|
||||||
topic_slug: railiance
|
topic_slug: railiance
|
||||||
planning_priority: medium
|
planning_priority: medium
|
||||||
created: "2026-05-19"
|
created: "2026-05-19"
|
||||||
updated: "2026-05-19"
|
updated: "2026-05-22"
|
||||||
state_hub_workstream_id: "b61a9aca-4e43-4b3d-a48b-999e0fa842cf"
|
state_hub_workstream_id: "b61a9aca-4e43-4b3d-a48b-999e0fa842cf"
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -18,14 +18,15 @@ state_hub_workstream_id: "b61a9aca-4e43-4b3d-a48b-999e0fa842cf"
|
|||||||
This workplan collects concrete follow-ups surfaced while shipping
|
This workplan collects concrete follow-ups surfaced while shipping
|
||||||
`vergabe-teilnahme` under `RAILIANCE-WP-0002`. Each item is small,
|
`vergabe-teilnahme` under `RAILIANCE-WP-0002`. Each item is small,
|
||||||
independent, and can be picked up in isolation when the next S5 app
|
independent, and can be picked up in isolation when the next S5 app
|
||||||
lands or when the next operator onboards. Status is `backlog` —
|
lands or when the next operator onboards. Activated on 2026-05-22;
|
||||||
nothing here is blocking the live deployment.
|
local railiance-apps guardrails are implemented, with the package
|
||||||
|
publication item blocked on sibling-repo release work.
|
||||||
|
|
||||||
## I01 — URL-encode DB passwords at Secret-build time
|
## I01 — URL-encode DB passwords at Secret-build time
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0004-I01
|
id: RAILIANCE-WP-0004-I01
|
||||||
status: todo
|
status: done
|
||||||
priority: medium
|
priority: medium
|
||||||
state_hub_task_id: "a05a855a-00a0-4e0e-ba82-27e0a072f777"
|
state_hub_task_id: "a05a855a-00a0-4e0e-ba82-27e0a072f777"
|
||||||
```
|
```
|
||||||
@@ -47,13 +48,17 @@ parsing is needed at all.
|
|||||||
**Where it lives:** new `tools/` script + Makefile target, or chart
|
**Where it lives:** new `tools/` script + Makefile target, or chart
|
||||||
helper template.
|
helper template.
|
||||||
|
|
||||||
|
**Implemented 2026-05-22.** Added `tools/build-database-url-secret.sh`
|
||||||
|
and `make vergabe-db-url-secret`; updated the app runbook to use the
|
||||||
|
helper during DB password rotation.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## I02 — Document the Django + kube-probe Host-header pattern
|
## I02 — Document the Django + kube-probe Host-header pattern
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0004-I02
|
id: RAILIANCE-WP-0004-I02
|
||||||
status: todo
|
status: done
|
||||||
priority: low
|
priority: low
|
||||||
state_hub_task_id: "22a212e6-31b1-490a-8d1c-0a33ddc62501"
|
state_hub_task_id: "22a212e6-31b1-490a-8d1c-0a33ddc62501"
|
||||||
```
|
```
|
||||||
@@ -71,13 +76,16 @@ pattern into a documented "Django-on-Railiance" recipe (short doc in
|
|||||||
the gotcha. Also worth a "common chart values" sketch if a second
|
the gotcha. Also worth a "common chart values" sketch if a second
|
||||||
Django app justifies the abstraction.
|
Django app justifies the abstraction.
|
||||||
|
|
||||||
|
**Implemented 2026-05-22.** Added `docs/django-on-railiance.md` and
|
||||||
|
cross-linked it from the `vergabe-teilnahme` runbook.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## I03 — Publish `issue-core` to a Gitea Python package registry
|
## I03 — Publish `issue-core` to a Gitea Python package registry
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0004-I03
|
id: RAILIANCE-WP-0004-I03
|
||||||
status: todo
|
status: blocked
|
||||||
priority: medium
|
priority: medium
|
||||||
state_hub_task_id: "f412b874-0670-4a4a-89fc-575fe4994646"
|
state_hub_task_id: "f412b874-0670-4a4a-89fc-575fe4994646"
|
||||||
```
|
```
|
||||||
@@ -98,13 +106,19 @@ drops the `--build-context` and the build becomes portable.
|
|||||||
(small Helm values change) and a release pipeline for `issue-core`
|
(small Helm values change) and a release pipeline for `issue-core`
|
||||||
(separate repo).
|
(separate repo).
|
||||||
|
|
||||||
|
**Local progress 2026-05-22.** `helm/gitea-registry-values.yaml` now
|
||||||
|
sets `packages.LIMIT_SIZE_PYPI: -1`, and
|
||||||
|
`docs/gitea-package-registry.md` documents the Gitea PyPI endpoint plus
|
||||||
|
the `issue-core` migration. The remaining release and dependency change
|
||||||
|
must happen in the `issue-core` and `vergabe-teilnahme` repos.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## I04 — Operator onboarding: install the `kubectl cnpg` plugin
|
## I04 — Operator onboarding: install the `kubectl cnpg` plugin
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0004-I04
|
id: RAILIANCE-WP-0004-I04
|
||||||
status: todo
|
status: done
|
||||||
priority: low
|
priority: low
|
||||||
state_hub_task_id: "2f44cad1-b70c-4406-91a9-0c0fa9c75583"
|
state_hub_task_id: "2f44cad1-b70c-4406-91a9-0c0fa9c75583"
|
||||||
```
|
```
|
||||||
@@ -120,13 +134,17 @@ line: `kubectl krew install cnpg` or a direct binary download). Add
|
|||||||
a `make check-tools` target that warns when `kubectl cnpg` or `helm`
|
a `make check-tools` target that warns when `kubectl cnpg` or `helm`
|
||||||
is missing.
|
is missing.
|
||||||
|
|
||||||
|
**Implemented 2026-05-22.** Added `make check-tools`,
|
||||||
|
`docs/operator-setup.md`, and cnpg fallback status output for Gitea and
|
||||||
|
the shared `apps-pg` cluster.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## I05 — Operator onboarding: SOPS / age key bootstrap
|
## I05 — Operator onboarding: SOPS / age key bootstrap
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0004-I05
|
id: RAILIANCE-WP-0004-I05
|
||||||
status: todo
|
status: done
|
||||||
priority: low
|
priority: low
|
||||||
state_hub_task_id: "741d8a73-8cb0-40ac-a218-f1d3a74ebef3"
|
state_hub_task_id: "741d8a73-8cb0-40ac-a218-f1d3a74ebef3"
|
||||||
```
|
```
|
||||||
@@ -143,13 +161,17 @@ procedure (where to put the key, how to verify, how to rotate). A
|
|||||||
decrypt a known sentinel would catch this at the first deploy attempt
|
decrypt a known sentinel would catch this at the first deploy attempt
|
||||||
rather than at the failing apply.
|
rather than at the failing apply.
|
||||||
|
|
||||||
|
**Implemented 2026-05-22.** Added `docs/operator-setup.md`,
|
||||||
|
`tools/check-sops.sh`, and `make check-sops` using
|
||||||
|
`helm/gitea-values.sops.yaml` as the sentinel by default.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## I06 — CI guard against stale committed manifests vs live CRD drift
|
## I06 — CI guard against stale committed manifests vs live CRD drift
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0004-I06
|
id: RAILIANCE-WP-0004-I06
|
||||||
status: todo
|
status: done
|
||||||
priority: medium
|
priority: medium
|
||||||
state_hub_task_id: "a319c20b-993c-46b7-889a-f0ac738056c4"
|
state_hub_task_id: "a319c20b-993c-46b7-889a-f0ac738056c4"
|
||||||
```
|
```
|
||||||
@@ -172,13 +194,18 @@ releases; strict server-side decoding catches drift that
|
|||||||
concern, but mirrored here because every S5 manifest in
|
concern, but mirrored here because every S5 manifest in
|
||||||
`charts/` and `manifests/` carries the same risk.
|
`charts/` and `manifests/` carries the same risk.
|
||||||
|
|
||||||
|
**Implemented 2026-05-22.** Added `tools/k8s-server-dry-run.sh`,
|
||||||
|
`make k8s-server-dry-run`, and a `.gitea/workflows/` PR workflow that
|
||||||
|
runs the guard when charts, Helm values, manifests, or the dry-run tool
|
||||||
|
change.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## I07 — `kubectl run --rm -i` smoke pattern is unreliable
|
## I07 — `kubectl run --rm -i` smoke pattern is unreliable
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0004-I07
|
id: RAILIANCE-WP-0004-I07
|
||||||
status: todo
|
status: done
|
||||||
priority: low
|
priority: low
|
||||||
state_hub_task_id: "e3f59b3d-95c8-4cf9-9943-b1597954fd77"
|
state_hub_task_id: "e3f59b3d-95c8-4cf9-9943-b1597954fd77"
|
||||||
```
|
```
|
||||||
@@ -195,13 +222,15 @@ runbook) recommending the persistent-pod-plus-exec pattern for any
|
|||||||
service-IP smoke check. Optional: ship `tools/smoke.sh` that
|
service-IP smoke check. Optional: ship `tools/smoke.sh` that
|
||||||
wraps the pattern.
|
wraps the pattern.
|
||||||
|
|
||||||
|
**Implemented 2026-05-22.** Added `docs/operator-recipes.md` and
|
||||||
|
`tools/smoke-service.sh`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
- Items are individually `todo`; the workplan status is `backlog` so
|
- Items were activated on 2026-05-22. Local railiance-apps pieces are
|
||||||
they don't show up in active-workstream lists. Promote an item to
|
complete except I03, which is blocked on sibling-repo release work.
|
||||||
`active` (and its tasks to `in_progress`) when you pick it up.
|
|
||||||
- I06 is genuinely cross-repo; the others are local to
|
- I06 is genuinely cross-repo; the others are local to
|
||||||
`railiance-apps` or its operator workflow.
|
`railiance-apps` or its operator workflow.
|
||||||
- The first three items (I01, I02, I03) are the highest-leverage
|
- The first three items (I01, I02, I03) are the highest-leverage
|
||||||
|
|||||||
Reference in New Issue
Block a user