fix: harden npm gateway bootstrap and add coldstart stress validation

Author: zhaojing1987

apphub/src/services/integration_session_bridge.py

@@ -1,6 +1,7 @@
 import os
 import re
 import json
+import time
 from pathlib import Path
 from typing import Literal, Optional
 from urllib.parse import urlparse
@@ -39,6 +40,8 @@ def __init__(self, gateway_origin: Optional[str] = None):
         ).rstrip("/")
         self.npm_credential_path = Path(os.getenv("WEBSOFT9_NPM_CREDENTIAL_PATH", str(data_root / "credential.json")))
         self.npm_database_path = Path(os.getenv("WEBSOFT9_NPM_DATABASE_PATH", str(data_root / "database.sqlite")))
+        self.npm_bootstrap_retry_attempts = max(1, int(os.getenv("WEBSOFT9_NPM_BOOTSTRAP_RETRY_ATTEMPTS", "6")))
+        self.npm_bootstrap_retry_interval_seconds = max(1, int(os.getenv("WEBSOFT9_NPM_BOOTSTRAP_RETRY_INTERVAL_SECONDS", "2")))
 
     def _resolve_platform_origin(self) -> str:
         public_origin = (os.getenv("WEBSOFT9_PLATFORM_PUBLIC_ORIGIN") or "").strip().rstrip("/")
@@ -273,7 +276,7 @@ def bootstrap_npm(self) -> list[dict[str, object]]:
             active_credentials = self.credential_provider.get_npm_credentials()
 
             fallback_credentials = self.credential_provider.get_npm_config_credentials()
-            response = self._request_npm_token(session, active_credentials.username, active_credentials.password)
+            response = self._request_npm_token_with_retry(session, active_credentials.username, active_credentials.password)
 
             should_try_fallback = (
                 response.status_code in {401, 403}
@@ -287,7 +290,7 @@ def bootstrap_npm(self) -> list[dict[str, object]]:
             )
 
             if should_try_fallback:
-                fallback_response = self._request_npm_token(session, fallback_credentials.username, fallback_credentials.password)
+                fallback_response = self._request_npm_token_with_retry(session, fallback_credentials.username, fallback_credentials.password)
                 if fallback_response.status_code == 200:
                     self.credential_provider.write_npm_credentials(fallback_credentials)
                     active_credentials = fallback_credentials
@@ -298,7 +301,7 @@ def bootstrap_npm(self) -> list[dict[str, object]]:
 
             if response.status_code in {401, 403} and self.credential_provider.sync_npm_credentials(active_credentials):
                 logger.warning("Nginx Proxy Manager authentication recovered after synchronizing stored credentials into the runtime database")
-                response = self._request_npm_token(session, active_credentials.username, active_credentials.password)
+                response = self._request_npm_token_with_retry(session, active_credentials.username, active_credentials.password)
 
             if response.status_code in {401, 403}:
                 raise CustomException(
@@ -348,6 +351,45 @@ def _request_npm_token(self, session: requests.Session, username: str, password:
             timeout=20,
         )
 
+    def _request_npm_token_with_retry(self, session: requests.Session, username: str, password: str) -> requests.Response:
+        last_exception: Exception | None = None
+        response: requests.Response | None = None
+
+        for attempt in range(1, self.npm_bootstrap_retry_attempts + 1):
+            try:
+                response = self._request_npm_token(session, username, password)
+                if response.status_code not in {502, 503, 504}:
+                    return response
+
+                if attempt < self.npm_bootstrap_retry_attempts:
+                    logger.warning(
+                        "Nginx Proxy Manager token endpoint not ready "
+                        f"(status={response.status_code}, attempt={attempt}/{self.npm_bootstrap_retry_attempts}); retrying"
+                    )
+                    time.sleep(self.npm_bootstrap_retry_interval_seconds)
+            except requests.RequestException as exc:
+                last_exception = exc
+                if attempt < self.npm_bootstrap_retry_attempts:
+                    logger.warning(
+                        "Nginx Proxy Manager token request failed "
+                        f"(attempt={attempt}/{self.npm_bootstrap_retry_attempts}): {exc}; retrying"
+                    )
+                    time.sleep(self.npm_bootstrap_retry_interval_seconds)
+                else:
+                    break
+
+        if response is not None:
+            return response
+
+        if last_exception is not None:
+            raise last_exception
+
+        raise CustomException(
+            status_code=502,
+            message="Integration Session Bootstrap Failed",
+            details="Unable to restore the embedded gateway session automatically",
+        )
+
     def _request_portainer_token(self, session: requests.Session, username: str, password: str) -> requests.Response:
         return session.post(
             f"{self.portainer_direct_origin}/api/auth",

apphub/tests/test_integrations.py

@@ -141,7 +141,8 @@ def post(self, url, json, timeout):
 
     cookies = bridge.bootstrap_portainer()
 
-    assert cookies == [{"name": bridge._portainer_cookie_name(), "value": "portainer-token", "path": "/", "httponly": True}]
+    assert cookies[0] == {"name": bridge._portainer_cookie_name(), "value": "portainer-token", "path": "/", "httponly": True}
+    assert {cookie["name"] for cookie in cookies[1:]} == {"portainer_jwt", "portainer.JWT", "portainer_api_key"}
     assert writes == [fallback]
     assert len(fake_session.calls) == 2
 
@@ -192,4 +193,45 @@ def post(self, url, json, timeout):
     assert writes == [fallback]
     assert len(fake_session.calls) == 2
     assert fake_session.calls[0][0] == "http://127.0.0.1:81/api/tokens"
-    assert fake_session.calls[1][0] == "http://127.0.0.1:81/api/tokens"
+    assert fake_session.calls[1][0] == "http://127.0.0.1:81/api/tokens"
+
+
+def test_npm_bootstrap_retries_transient_gateway_errors(monkeypatch):
+    bridge = IntegrationSessionBridge(gateway_origin="http://gateway.test")
+    bridge.npm_bootstrap_retry_attempts = 3
+    bridge.npm_bootstrap_retry_interval_seconds = 1
+
+    credentials = SimpleNamespace(
+        username="admin@example.com",
+        password="active-pass",
+        nickname="operator",
+        display_name="Operator",
+    )
+
+    bridge.credential_provider = SimpleNamespace(
+        get_npm_credentials=lambda: credentials,
+        get_npm_config_credentials=lambda: credentials,
+        write_npm_credentials=lambda _credentials: None,
+        sync_npm_credentials=lambda _credentials: False,
+    )
+
+    call_count = {"value": 0}
+
+    def fake_request_npm_token(_session, username, password):
+        call_count["value"] += 1
+        assert username == "admin@example.com"
+        assert password == "active-pass"
+        if call_count["value"] == 1:
+            return FakeResponse(502, {"message": "Bad Gateway"})
+        return FakeResponse(200, {"token": "npm-token"})
+
+    monkeypatch.setattr(bridge, "_request_npm_token", fake_request_npm_token)
+    monkeypatch.setattr("src.services.integration_session_bridge.time.sleep", lambda _seconds: None)
+
+    cookies = bridge.bootstrap_npm()
+
+    assert call_count["value"] == 2
+    assert cookies == [
+        {"name": bridge._npm_token_cookie_name(), "value": "npm-token", "path": "/", "httponly": False},
+        {"name": bridge._npm_nickname_cookie_name(), "value": "operator", "path": "/", "httponly": False},
+    ]

install/lib/upgrade-legacy.sh

@@ -746,6 +746,19 @@ run_upgrade_legacy() {
   # Prepare modern deployment material without touching the transformed data root.
   install_prepare_material "$install_path" "$image_tag" "$console_port"
 
+  # Resolve the modern container name from generated deployment material.
+  # Legacy migration can run on dev/rc channels where container name is not "websoft9".
+  local env_file compose_file
+  env_file="${install_path}/.env"
+  compose_file="${install_path}/docker-compose.yml"
+  if [ -f "$env_file" ]; then
+    CONTAINER_NAME="$(grep -m1 '^CONTAINER_NAME=' "$env_file" 2>/dev/null | cut -d= -f2-)"
+    [ -n "$CONTAINER_NAME" ] && export CONTAINER_NAME
+  fi
+  MODERN_CONTAINER_NAME="$(_resolve_container_name "$compose_file")"
+  export MODERN_CONTAINER_NAME
+  log_info "Resolved modern container name for migration: ${MODERN_CONTAINER_NAME}"
+
   if [ "${W9_DRY_RUN:-0}" = "1" ]; then
     log_info "(dry-run) pre-cutover migration steps completed; stopping before the modern runtime takeover"
     log_info "Pre-migration backup point: $backup_dir"
@@ -781,13 +794,10 @@ run_upgrade_legacy() {
     die "$EXIT_VALIDATE" "Migration failed during post-cutover validation"
   fi
 
-  # Stage 8b: remove legacy containers and volumes.
-  # Host-level artifacts (/data/compose, /data/apps, Cockpit, systemd) are
-  # deliberately retained — legacy stacks still reference /data/compose for
-  # bind mounts, and control-plane cleanup should happen after the rollback
-  # window closes.
-  log_step "Removing legacy containers and volumes"
-  _uninstall_legacy "purge" "0" "1" "0"
+  # Stage 8b: remove legacy containers, volumes, and control-plane artifacts.
+  # On successful migration, fully retire Cockpit/systemd-based legacy runtime.
+  log_step "Removing legacy containers, volumes, and legacy control plane"
+  _uninstall_legacy "purge" "0" "1" "1"
 
   log_info "==== Legacy-to-modern migration completed successfully ===="
   print_runtime_summary migration "$install_path" "$console_port" "$backup_dir"

scripts/npm_gateway_coldstart_stress.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+container_name="${1:-websoft9-dev}"
+rounds="${2:-10}"
+settle_seconds="${3:-1}"
+report_file="${4:-/tmp/websoft9_npm_coldstart_report.txt}"
+health_wait_attempts="${WEBSOFT9_STRESS_HEALTH_WAIT_ATTEMPTS:-30}"
+
+if ! [[ "$rounds" =~ ^[0-9]+$ ]] || [[ "$rounds" -le 0 ]]; then
+  echo "rounds must be a positive integer" >&2
+  exit 1
+fi
+
+if ! [[ "$settle_seconds" =~ ^[0-9]+$ ]] || [[ "$settle_seconds" -lt 0 ]]; then
+  echo "settle_seconds must be a non-negative integer" >&2
+  exit 1
+fi
+
+api_url="http://127.0.0.1:9000/api/integrations/npm/session"
+health_url="http://127.0.0.1:9000/api/healthz"
+start_ts="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+
+curl_common=(--noproxy '*' --silent --show-error --connect-timeout 2 --max-time 4)
+
+declare -a duration_ms_list=()
+success_count=0
+failure_count=0
+
+: > "$report_file"
+echo "websoft9 npm gateway coldstart stress report" >> "$report_file"
+echo "start_time_utc=$start_ts" >> "$report_file"
+echo "container=$container_name rounds=$rounds settle_seconds=$settle_seconds" >> "$report_file"
+echo "health_wait_attempts=$health_wait_attempts" >> "$report_file"
+echo "" >> "$report_file"
+
+echo "Running $rounds rounds against $container_name"
+
+for ((i=1; i<=rounds; i++)); do
+  echo "[$i/$rounds] restarting npm-backend + npm-nginx"
+  docker exec "$container_name" supervisorctl -c /etc/supervisor/conf.d/websoft9-platform.conf restart npm-backend npm-nginx >/tmp/w9_supervisor_restart.log 2>&1
+
+  if [[ "$settle_seconds" -gt 0 ]]; then
+    sleep "$settle_seconds"
+  fi
+
+  health_ready="false"
+  health_recovery_ms=0
+  health_start_ms="$(date +%s%3N)"
+  for ((attempt=1; attempt<=health_wait_attempts; attempt++)); do
+    health_code="$(curl "${curl_common[@]}" -o /dev/null -w '%{http_code}' "$health_url" || true)"
+    if [[ "$health_code" == "200" ]]; then
+      health_ready="true"
+      health_end_ms="$(date +%s%3N)"
+      health_recovery_ms=$((health_end_ms - health_start_ms))
+      break
+    fi
+    sleep 1
+  done
+
+  response_file="/tmp/w9_npm_stress_resp_${i}.json"
+  timing_file="/tmp/w9_npm_stress_timing_${i}.txt"
+  : > "$response_file"
+  : > "$timing_file"
+
+  if [[ "$health_ready" == "true" ]]; then
+    curl "${curl_common[@]}" -X POST "$api_url" -H 'Content-Type: application/json' -o "$response_file" -w '%{http_code} %{time_total}\n' > "$timing_file" || true
+  else
+    printf '000 0\n' > "$timing_file"
+    printf '{"status":"error","details":"gateway health check did not recover in time"}' > "$response_file"
+  fi
+
+  status_code="$(awk '{print $1}' "$timing_file" | tr -d '\r' || true)"
+  time_total="$(awk '{print $2}' "$timing_file" | tr -d '\r' || true)"
+
+  if [[ -z "$status_code" ]]; then
+    status_code="000"
+  fi
+
+  duration_ms="$(awk -v t="${time_total:-0}" 'BEGIN{printf "%d", t*1000}')"
+  if [[ -z "$duration_ms" ]]; then
+    duration_ms=0
+  fi
+
+  if [[ "$status_code" == "200" ]]; then
+    result="PASS"
+    success_count=$((success_count + 1))
+  else
+    result="FAIL"
+    failure_count=$((failure_count + 1))
+  fi
+
+  duration_ms_list+=("$duration_ms")
+  payload_preview="$(tr -d '\n' < "$response_file" | cut -c1-200)"
+
+  printf 'round=%02d result=%s health_ready=%s health_recovery_ms=%s status=%s duration_ms=%s payload=%s\n' "$i" "$result" "$health_ready" "$health_recovery_ms" "$status_code" "$duration_ms" "$payload_preview" | tee -a "$report_file"
+done
+
+sorted_durations="$(printf '%s\n' "${duration_ms_list[@]}" | sort -n)"
+mean_ms="$(printf '%s\n' "${duration_ms_list[@]}" | awk '{sum+=$1} END{if (NR>0) printf "%.2f", sum/NR; else print "0"}')"
+min_ms="$(printf '%s\n' "$sorted_durations" | head -n 1)"
+max_ms="$(printf '%s\n' "$sorted_durations" | tail -n 1)"
+
+p95_index=$(( (rounds * 95 + 99) / 100 ))
+p95_ms="$(printf '%s\n' "$sorted_durations" | sed -n "${p95_index}p")"
+
+success_rate="$(awk -v s="$success_count" -v r="$rounds" 'BEGIN{printf "%.2f", (r>0 ? s*100/r : 0)}')"
+end_ts="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+
+echo "" >> "$report_file"
+echo "summary:" >> "$report_file"
+echo "end_time_utc=$end_ts" >> "$report_file"
+echo "success_count=$success_count failure_count=$failure_count success_rate_percent=$success_rate" >> "$report_file"
+echo "latency_ms min=$min_ms mean=$mean_ms p95=$p95_ms max=$max_ms" >> "$report_file"
+
+echo ""
+echo "Completed. Summary:"
+cat "$report_file" | tail -n 5
+
+echo "Report saved to $report_file"

scripts/platform-entrypoint.sh

@@ -238,7 +238,7 @@ bootstrap_portainer() {
 bootstrap_nginx_proxy_manager() {
   log "phase=workspace-bootstrap action=bootstrap-nginx-proxy-manager"
 
-  if ! wait_for_url "nginx-proxy-manager" "${WEBSOFT9_NPM_HEALTH_URL:-http://127.0.0.1:81/}" 45; then
+  if ! wait_for_url "nginx-proxy-manager" "${WEBSOFT9_NPM_HEALTH_URL:-http://127.0.0.1:81/api/}" 45; then
     write_status "degraded" "nginx-proxy-manager failed to become healthy during bootstrap"
     return 0
   fi