fix: Update env-validator.py to handle dynamic variable discovery #85
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to AWS App Runner | |
| on: | |
| push: | |
| branches: [ main, master ] | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Deployment environment' | |
| required: true | |
| default: 'production' | |
| type: choice | |
| options: | |
| - production | |
| - staging | |
| # Project-specific environment variables | |
| # TODO: Add your project's secrets here | |
| env: | |
| # Example secrets (uncomment and modify for your project): | |
| # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || '' }} | |
| # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY || '' }} | |
| # VERTEC_API_KEY: ${{ secrets.VERTEC_API_KEY || '' }} | |
| # Data access for build (if downloading data from S3) | |
| MXCP_DATA_ACCESS_KEY_ID: ${{ secrets.MXCP_DATA_ACCESS_KEY_ID || '' }} | |
| MXCP_DATA_SECRET_ACCESS_KEY: ${{ secrets.MXCP_DATA_SECRET_ACCESS_KEY || '' }} | |
| jobs: | |
| check-secrets: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| aws-creds-configured: ${{ steps.check-aws.outputs.configured }} | |
| steps: | |
| - id: check-aws | |
| run: | | |
| if [[ -n "${{ secrets.AWS_ACCESS_KEY_ID }}" ]] && [[ -n "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then | |
| echo "configured=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "configured=false" >> $GITHUB_OUTPUT | |
| echo "⚠️ AWS credentials not configured. Please add AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to repository secrets." | |
| fi | |
| pre-flight-checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Validate RAW deployment configuration | |
| run: | | |
| echo "🔍 Validating RAW Labs deployment configuration..." | |
| # Load configuration from config.env and GitHub Variables | |
| if [ -f "deployment/config.env" ]; then | |
| set -a | |
| source deployment/config.env | |
| set +a | |
| echo "✅ Loaded defaults from deployment/config.env" | |
| fi | |
| # Override with GitHub Variables if set | |
| [ -n "${{ vars.AWS_REGION }}" ] && AWS_REGION="${{ vars.AWS_REGION }}" | |
| [ -n "${{ vars.AWS_ACCOUNT_ID }}" ] && AWS_ACCOUNT_ID="${{ vars.AWS_ACCOUNT_ID }}" | |
| [ -n "${{ vars.ECR_REPOSITORY }}" ] && ECR_REPOSITORY="${{ vars.ECR_REPOSITORY }}" | |
| [ -n "${{ vars.APP_RUNNER_SERVICE }}" ] && APP_RUNNER_SERVICE="${{ vars.APP_RUNNER_SERVICE }}" | |
| # Validate required values exist (from either source) | |
| if [ -z "$AWS_ACCOUNT_ID" ]; then | |
| echo "❌ AWS_ACCOUNT_ID not set (neither in config.env nor GitHub Variables)" | |
| exit 1 | |
| fi | |
| if [ -z "$AWS_REGION" ]; then | |
| echo "❌ AWS_REGION not set (neither in config.env nor GitHub Variables)" | |
| exit 1 | |
| fi | |
| if [ -z "$ECR_REPOSITORY" ]; then | |
| echo "❌ ECR_REPOSITORY not set (neither in config.env nor GitHub Variables)" | |
| exit 1 | |
| fi | |
| if [ -z "$APP_RUNNER_SERVICE" ]; then | |
| echo "❌ APP_RUNNER_SERVICE not set (neither in config.env nor GitHub Variables)" | |
| exit 1 | |
| fi | |
| # RAW-specific validations | |
| if [ "$AWS_ACCOUNT_ID" != "684130658470" ]; then | |
| echo "⚠️ Warning: AWS_ACCOUNT_ID is not the standard RAW Labs account (684130658470)" | |
| echo " Current value: $AWS_ACCOUNT_ID" | |
| fi | |
| if [ "$AWS_REGION" != "eu-west-1" ]; then | |
| echo "⚠️ Warning: AWS_REGION is not the standard RAW Labs region (eu-west-1)" | |
| echo " Current value: $AWS_REGION" | |
| fi | |
| echo "✅ All required configuration values are set" | |
| echo " Source: deployment/config.env + GitHub Variables overrides" | |
| - name: Validate YAML configurations | |
| run: | | |
| echo "📝 Validating YAML configurations..." | |
| python -c "import yaml; yaml.safe_load(open('mxcp-site.yml')); print('✅ mxcp-site.yml is valid')" | |
| python -c "import yaml, glob; [yaml.safe_load(open(f)) for f in glob.glob('tools/*.yml')]; print('✅ All tool configurations valid')" | |
| - name: Validate environment consistency | |
| run: | | |
| echo "🔍 Checking environment variable consistency..." | |
| python deployment/env-validator.py | |
| echo "✅ Pre-flight checks passed" | |
| deploy: | |
| needs: [check-secrets, pre-flight-checks] | |
| if: needs.check-secrets.outputs.aws-creds-configured == 'true' | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.environment || 'production' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Load configuration | |
| run: | | |
| echo "📋 Loading configuration..." | |
| # First, source defaults from config.env | |
| if [ -f "deployment/config.env" ]; then | |
| set -a # automatically export all variables | |
| source deployment/config.env | |
| set +a | |
| echo "✅ Loaded defaults from deployment/config.env" | |
| fi | |
| # Then, override with GitHub Variables if set | |
| if [ -n "${{ vars.AWS_REGION }}" ]; then | |
| export AWS_REGION="${{ vars.AWS_REGION }}" | |
| echo "📝 AWS_REGION overridden by GitHub Variable: $AWS_REGION" | |
| fi | |
| if [ -n "${{ vars.AWS_ACCOUNT_ID }}" ]; then | |
| export AWS_ACCOUNT_ID="${{ vars.AWS_ACCOUNT_ID }}" | |
| echo "📝 AWS_ACCOUNT_ID overridden by GitHub Variable: $AWS_ACCOUNT_ID" | |
| fi | |
| if [ -n "${{ vars.ECR_REPOSITORY }}" ]; then | |
| export ECR_REPOSITORY="${{ vars.ECR_REPOSITORY }}" | |
| echo "📝 ECR_REPOSITORY overridden by GitHub Variable: $ECR_REPOSITORY" | |
| fi | |
| if [ -n "${{ vars.APP_RUNNER_SERVICE }}" ]; then | |
| export APP_RUNNER_SERVICE="${{ vars.APP_RUNNER_SERVICE }}" | |
| echo "📝 APP_RUNNER_SERVICE overridden by GitHub Variable: $APP_RUNNER_SERVICE" | |
| fi | |
| if [ -n "${{ vars.CPU_SIZE }}" ]; then | |
| export CPU_SIZE="${{ vars.CPU_SIZE }}" | |
| echo "📝 CPU_SIZE overridden by GitHub Variable: $CPU_SIZE" | |
| fi | |
| if [ -n "${{ vars.MEMORY_SIZE }}" ]; then | |
| export MEMORY_SIZE="${{ vars.MEMORY_SIZE }}" | |
| echo "📝 MEMORY_SIZE overridden by GitHub Variable: $MEMORY_SIZE" | |
| fi | |
| # Export to GITHUB_ENV for subsequent steps | |
| echo "AWS_REGION=$AWS_REGION" >> $GITHUB_ENV | |
| echo "AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID" >> $GITHUB_ENV | |
| echo "ECR_REPOSITORY=$ECR_REPOSITORY" >> $GITHUB_ENV | |
| echo "APP_RUNNER_SERVICE=$APP_RUNNER_SERVICE" >> $GITHUB_ENV | |
| echo "CPU_SIZE=$CPU_SIZE" >> $GITHUB_ENV | |
| echo "MEMORY_SIZE=$MEMORY_SIZE" >> $GITHUB_ENV | |
| echo "📊 Final configuration:" | |
| echo " AWS_REGION=$AWS_REGION" | |
| echo " AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID" | |
| echo " ECR_REPOSITORY=$ECR_REPOSITORY" | |
| echo " APP_RUNNER_SERVICE=$APP_RUNNER_SERVICE" | |
| echo " CPU_SIZE=$CPU_SIZE" | |
| echo " MEMORY_SIZE=$MEMORY_SIZE" | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Set up environment | |
| run: | | |
| # Create config file for App Runner deployment script (already have values in env) | |
| echo "AWS_ACCOUNT_ID=${{ env.AWS_ACCOUNT_ID }}" > .github/config.env | |
| echo "AWS_REGION=${{ env.AWS_REGION }}" >> .github/config.env | |
| echo "SERVICE_NAME=${{ env.APP_RUNNER_SERVICE }}" >> .github/config.env | |
| echo "ECR_REPOSITORY=${{ env.ECR_REPOSITORY }}" >> .github/config.env | |
| echo "CPU_SIZE=\"${{ env.CPU_SIZE }}\"" >> .github/config.env | |
| echo "MEMORY_SIZE=\"${{ env.MEMORY_SIZE }}\"" >> .github/config.env | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install just | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin | |
| echo "/usr/local/bin" >> $GITHUB_PATH | |
| - name: Install dependencies | |
| run: | | |
| # Install project requirements if they exist | |
| if [ -f requirements.txt ]; then | |
| echo "📦 Installing project dependencies..." | |
| pip install -r requirements.txt | |
| fi | |
| # Install deployment requirements | |
| pip install -r deployment/requirements.txt | |
| - name: Prepare data before build | |
| run: | | |
| echo "📥 Preparing data outside Docker..." | |
| # Export credentials for data download (if needed) | |
| export AWS_ACCESS_KEY_ID=${{ secrets.MXCP_DATA_ACCESS_KEY_ID }} | |
| export AWS_SECRET_ACCESS_KEY=${{ secrets.MXCP_DATA_SECRET_ACCESS_KEY }} | |
| # Download data only (no dbt needed here) | |
| # This will be a no-op for API projects (just creates directories) | |
| just prepare-data | |
| echo "✅ Data preparation complete" | |
| - name: Build Docker image and prepare ECR | |
| run: | | |
| # Login to ECR | |
| aws ecr get-login-password --region ${{ env.AWS_REGION }} | \ | |
| docker login --username AWS --password-stdin ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com | |
| # Check if ECR repository exists (create only if we have permissions) | |
| if aws ecr describe-repositories --repository-names ${{ env.ECR_REPOSITORY }} --region ${{ env.AWS_REGION }} 2>/dev/null; then | |
| echo "✅ ECR repository ${{ env.ECR_REPOSITORY }} exists" | |
| else | |
| echo "⚠️ ECR repository ${{ env.ECR_REPOSITORY }} doesn't exist" | |
| # Try to create it, but don't fail if we lack permissions (external teams use shared repos) | |
| if aws ecr create-repository --repository-name ${{ env.ECR_REPOSITORY }} --region ${{ env.AWS_REGION }} --image-scanning-configuration scanOnPush=true 2>/dev/null; then | |
| echo "✅ Created ECR repository ${{ env.ECR_REPOSITORY }}" | |
| else | |
| echo "⚠️ Cannot create repository (insufficient permissions). Assuming it's a shared repository that should already exist." | |
| # Verify the repository actually exists by trying to describe it again | |
| aws ecr describe-repositories --repository-names ${{ env.ECR_REPOSITORY }} --region ${{ env.AWS_REGION }} || \ | |
| (echo "❌ ERROR: ECR repository ${{ env.ECR_REPOSITORY }} does not exist and cannot be created. Please contact your AWS administrator." && exit 1) | |
| fi | |
| fi | |
| # Build from project root with deployment/Dockerfile (no data credentials needed) | |
| docker build -f deployment/Dockerfile \ | |
| -t ${{ env.ECR_REPOSITORY }}:${{ github.sha }} . | |
| # Tag for ECR | |
| docker tag ${{ env.ECR_REPOSITORY }}:${{ github.sha }} ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}:${{ github.sha }} | |
| docker tag ${{ env.ECR_REPOSITORY }}:${{ github.sha }} ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}:latest | |
| - name: Test Docker image with secrets | |
| run: | | |
| echo "🧪 Running full test suite with secrets..." | |
| # Pass environment variables to Docker, but exclude HOME which conflicts with container's HOME | |
| env | grep -v '^HOME=' > test.env | |
| # Run tests with environment variables | |
| docker run --rm \ | |
| --env-file test.env \ | |
| ${{ env.ECR_REPOSITORY }}:${{ github.sha }} \ | |
| just test-all | |
| # Clean up | |
| rm -f test.env | |
| echo "✅ All tests passed!" | |
| - name: Push to ECR | |
| run: | | |
| echo "📤 Pushing tested image to ECR..." | |
| # Push to ECR | |
| docker push ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}:${{ github.sha }} | |
| docker push ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}:latest | |
| - name: Deploy to AWS App Runner | |
| run: | | |
| chmod +x .github/scripts/deploy-app-runner.sh | |
| ./.github/scripts/deploy-app-runner.sh | |
| - name: Monitor App Runner deployment | |
| run: | | |
| echo "📊 Monitoring App Runner service creation/update..." | |
| # Discover service ARN dynamically by name (ARN changes when service is recreated) | |
| echo "🔍 Finding service ARN for: ${{ env.APP_RUNNER_SERVICE }}" | |
| SERVICE_ARN=$(aws apprunner list-services --region ${{ env.AWS_REGION }} \ | |
| --query "ServiceSummaryList[?ServiceName=='${{ env.APP_RUNNER_SERVICE }}'].ServiceArn | [0]" \ | |
| --output text) | |
| if [ "$SERVICE_ARN" == "None" ] || [ -z "$SERVICE_ARN" ]; then | |
| echo "❌ Service not found after deployment" | |
| exit 1 | |
| fi | |
| echo "✅ Found service ARN: $SERVICE_ARN" | |
| # Monitor for up to 10 minutes (App Runner can take time) | |
| for i in {1..20}; do | |
| STATUS=$(aws apprunner describe-service --service-arn "$SERVICE_ARN" --query 'Service.Status' --output text) | |
| echo "[$i/20] Current status: $STATUS" | |
| case $STATUS in | |
| "RUNNING") | |
| echo "✅ Service is RUNNING!" | |
| break | |
| ;; | |
| "CREATE_FAILED"|"UPDATE_FAILED"|"DELETE_FAILED") | |
| echo "❌ Service failed with status: $STATUS" | |
| # Get more details about the failure | |
| aws apprunner describe-service --service-arn "$SERVICE_ARN" --query 'Service.{Status:Status,StatusMessage:StatusMessage}' --output table | |
| exit 1 | |
| ;; | |
| "OPERATION_IN_PROGRESS"|"CREATING"|"UPDATING") | |
| echo "⏳ Service still starting... waiting 30 seconds" | |
| sleep 30 | |
| ;; | |
| *) | |
| echo "⚠️ Unknown status: $STATUS" | |
| sleep 30 | |
| ;; | |
| esac | |
| done | |
| # Final status check | |
| FINAL_STATUS=$(aws apprunner describe-service --service-arn "$SERVICE_ARN" --query 'Service.Status' --output text) | |
| if [ "$FINAL_STATUS" != "RUNNING" ]; then | |
| echo "❌ Service did not reach RUNNING state after 10 minutes" | |
| echo "Final status: $FINAL_STATUS" | |
| aws apprunner describe-service --service-arn "$SERVICE_ARN" --query 'Service.{Status:Status,StatusMessage:StatusMessage}' --output table | |
| exit 1 | |
| fi | |
| - name: Verify deployment | |
| run: | | |
| # Wait for deployment to stabilize | |
| echo "⏳ Waiting for App Runner service to stabilize..." | |
| sleep 60 | |
| # Get service URL | |
| SERVICE_URL=$(aws apprunner describe-service \ | |
| --service-arn "arn:aws:apprunner:${{ env.AWS_REGION }}:${{ env.AWS_ACCOUNT_ID }}:service/${{ env.APP_RUNNER_SERVICE }}" \ | |
| --query 'Service.ServiceUrl' --output text) | |
| echo "🌐 Service URL: https://$SERVICE_URL" | |
| # Verify service health | |
| echo "🔍 Verifying service health..." | |
| if curl -f "https://$SERVICE_URL/health" > /dev/null 2>&1; then | |
| echo "✅ Health endpoint responding" | |
| else | |
| echo "⚠️ Health check failed, service may still be starting..." | |
| fi | |
| # Verify MCP endpoint | |
| echo "🔍 Verifying MCP endpoint..." | |
| if curl -f "https://$SERVICE_URL/mcp" > /dev/null 2>&1; then | |
| echo "✅ MCP endpoint accessible" | |
| else | |
| echo "⚠️ MCP endpoint not responding yet" | |
| fi | |
| echo "" | |
| echo "🎉 Deployment completed!" | |
| echo "🔗 Service available at: https://$SERVICE_URL/mcp" | |
| - name: Deployment summary | |
| if: always() | |
| run: | | |
| # Configuration from GitHub variables | |
| echo "## 🚀 Deployment Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Service**: ${{ env.APP_RUNNER_SERVICE }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Region**: ${{ env.AWS_REGION }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Environment**: ${{ github.event.inputs.environment || 'production' }}" >> $GITHUB_STEP_SUMMARY | |
| # Get service URL if deployment succeeded | |
| if SERVICE_URL=$(aws apprunner describe-service --service-arn "arn:aws:apprunner:${{ env.AWS_REGION }}:${{ env.AWS_ACCOUNT_ID }}:service/${{ env.APP_RUNNER_SERVICE }}" --query 'Service.ServiceUrl' --output text 2>/dev/null); then | |
| echo "- **Service URL**: https://$SERVICE_URL" >> $GITHUB_STEP_SUMMARY | |
| echo "- **MCP Endpoint**: https://$SERVICE_URL/mcp" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "- **Commit**: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY |