PEP458: Add minimal RSTUF service configuration (#15241)

* remove vault container

The vault container was a previous TUF implementation
(TUF initialization #7488)

The new integration with RSTUF does not require this container.

Signed-off-by: Kairo de Araujo <[email protected]>

* PEP 458: Add RSTUF services in the Warehouse Infra

This commit adds the RSTUF services to the Warehouse infrastructure
for development and sets the minimum required to start RSTUF services.

It adds the RSTUF API, which is used later to integrate into Warehouse
and RSTUF Worker, which is responsible for computing the TUF metadata.

The RSTUF requires the Postgres and Redis.
Postgres stores the rstuf database used for TUF metadata computing.
Redis stores the task message queue between RSTUF API and Worker, task
backend result, and live settings between RSTUF services.

RSTUF shares the same Postgres and Redis in development environment
but has a specific setup to use its own Postgres database and Redis
database ID.

Postgresql URI
`RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf`

Redis DB Broker and Result is id 1
`RSTUF_BROKER_SERVER=redis://redis/1`
`RSTUF_REDIS_SERVER_DB_RESULT=1`

Redis DB for TUF repository settings is 2
`RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2`

This commit also includes TUF database creation in the Makefile
during the `make initdb`.

Signed-off-by: Kairo de Araujo <[email protected]>

* remove rstuf-worker unnecessary settings

Remove settings from rstuf-worker in docker-compose.yml

Signed-off-by: Kairo de Araujo <[email protected]>

* remove vault volume from docker-compose

---------

Signed-off-by: Kairo de Araujo <[email protected]>
Co-authored-by: Ee Durbin <[email protected]>

Author: kairoaraujo

Makefile

@@ -101,6 +101,8 @@ initdb: .state/docker-build-base
 	docker compose run --rm web psql -h db -d postgres -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse';"
 	docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS warehouse"
 	docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE warehouse ENCODING 'UTF8'"
+	docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS rstuf"
+	docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE rstuf ENCODING 'UTF8'"
 	docker compose run --rm web bash -c "xz -d -f -k dev/$(DB).sql.xz --stdout | psql -h db -d warehouse -U postgres -v ON_ERROR_STOP=1 -1 -f -"
 	docker compose run --rm web psql -h db -d warehouse -U postgres -c "UPDATE users SET name='Ee Durbin' WHERE username='ewdurbin'"
 	$(MAKE) runmigrations

dev/environment

@@ -58,9 +58,6 @@ TOKEN_REMEMBER_DEVICE_SECRET="an insecure remember device auth secret key"
 
 WAREHOUSE_LEGACY_DOMAIN=pypi.python.org
 
-VAULT_URL="http://vault:8200"
-VAULT_TOKEN="an insecure vault access token"
-
 GITHUB_TOKEN_SCANNING_META_API_URL="http://notgithub:8000/meta/public_keys/token_scanning"
 TWOFACTORREQUIREMENT_ENABLED=true
 TWOFACTORMANDATE_AVAILABLE=true

dev/vault/config.hcl

@@ -6,28 +6,10 @@ volumes:
   packages-archive:
   sponsorlogos:
   policies:
-  vault:
   caches:
+  rstuf-metadata:
 
 services:
-  vault:
-    # NOTE: pinned for consistency with whats available in our deployment
-    image: vault:1.12.3
-    restart: on-failure
-    entrypoint: /bin/sh
-    command: /etc/vault/entry.sh
-    stop_signal: SIGINT
-    environment:
-      VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
-      VAULT_DEV_ROOT_TOKEN_ID: "an insecure vault access token"
-    ports:
-      - "8200:8200"
-    cap_add:
-      - IPC_LOCK
-    volumes:
-      - vault:/vault/file
-      - ./dev/vault:/etc/vault
-
   db:
     image: postgres:14.4
     ports:
@@ -160,6 +142,32 @@ services:
       ARCHIVE_FILES_BACKEND: "warehouse.packaging.services.LocalArchiveFileStorage path=/var/opt/warehouse/packages-archive/ url=http://files:9001/packages-archive/{path}"
       SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}"
 
+  rstuf-api:
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v0.9.0b1
+    ports:
+      - 8001:80
+    environment:
+      - RSTUF_BROKER_SERVER=redis://redis/1
+      - RSTUF_REDIS_SERVER=redis://redis
+      - RSTUF_REDIS_SERVER_DB_RESULT=1
+      - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
+
+  rstuf-worker:
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v0.11.0b1
+    volumes:
+      - rstuf-metadata:/var/opt/repository-service-tuf/storage
+    environment:
+      - RSTUF_STORAGE_BACKEND=LocalStorage
+      - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage
+      - RSTUF_BROKER_SERVER=redis://redis/1
+      - RSTUF_REDIS_SERVER=redis://redis
+      - RSTUF_REDIS_SERVER_DB_RESULT=1
+      - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
+      - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf
+    depends_on:
+      db:
+        condition: service_healthy
+
   static:
     build:
       context: .