deploy.yml

name: Deploy to AWS

on:
  workflow_dispatch:
    inputs:
      environment:
        description: 'Deployment environment'
        required: true
        default: 'staging'
        type: choice
        options:
          - staging
          - production
      force_deploy_without_wa:
        description: 'Deploy to production even if WhatsApp Business secrets are missing (emergency use only - bot will be offline)'
        required: false
        default: false
        type: boolean
  push:
    branches:
      - master
    paths-ignore:
      - '**.md'
      - 'docs/**'
      - '.claude/**'
      - 'Site Audit*/**'

concurrency:
  group: deploy-${{ github.event.inputs.environment || 'staging' }}
  cancel-in-progress: true

env:
  AWS_REGION: us-east-1
  NODE_VERSION: '20.x'
  PYTHON_VERSION: '3.11'

jobs:
  # ================================================================
  # STAGE 1: Lint, Test & Coverage Gate
  # ================================================================
  test:
    name: Lint & Test
    runs-on: ubuntu-latest
    outputs:
      coverage: ${{ steps.coverage.outputs.total }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup pnpm
        uses: pnpm/action-setup@v4

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Run linter
        run: pnpm run lint

      - name: Run tests with coverage
        run: pnpm run test:coverage
        env:
          DIDIT_API_KEY: test-placeholder
          DIDIT_WEBHOOK_SECRET: test-placeholder
          KYC_PROVIDER: didit

      - name: Extract coverage percentage
        id: coverage
        run: |
          TOTAL=$(cat coverage/coverage-summary.json 2>/dev/null \
            | jq -r '.total.lines.pct // 0' || echo "0")
          echo "total=$TOTAL" >> $GITHUB_OUTPUT
          echo "Coverage: ${TOTAL}%"

      - name: Enforce 80% coverage gate
        run: |
          COVERAGE="${{ steps.coverage.outputs.total }}"
          THRESHOLD=80
          if (( $(echo "$COVERAGE < $THRESHOLD" | bc -l 2>/dev/null || echo 1) )); then
            echo "::error::Coverage ${COVERAGE}% is below ${THRESHOLD}% threshold"
            exit 1
          fi

      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage/lcov.info
          flags: unittests
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false

  # ================================================================
  # STAGE 2: Security Scanning
  # ================================================================
  security-scan:
    name: Security Scan
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup pnpm
        uses: pnpm/action-setup@v4

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Dependency audit
        run: pnpm audit --audit-level=high || true

      - name: Check for known vulnerabilities
        run: |
          npx audit-ci --high 2>/dev/null || echo "::warning::Dependency vulnerabilities found - review before production"

      - name: Scan for hardcoded secrets
        run: |
          echo "Scanning for hardcoded secrets..."
          PATTERNS=(
            'password\s*=\s*["\x27][^"\x27]+'
            'api[_-]?key\s*=\s*["\x27][^"\x27]+'
            'secret\s*=\s*["\x27][^"\x27]+'
            'AKIA[0-9A-Z]{16}'
          )
          FOUND=0
          for pattern in "${PATTERNS[@]}"; do
            if grep -rn --include="*.ts" --include="*.js" --include="*.tsx" \
               -E "$pattern" services/ frontend/ 2>/dev/null | \
               grep -v node_modules | grep -v ".test." | grep -v ".spec." | \
               grep -v "process.env" | grep -v "secrets." | \
               grep -v ":\s*string" | grep -v "interface " | grep -v "\.d\.ts" | head -5; then
              FOUND=1
            fi
          done
          if [ "$FOUND" -eq 1 ]; then
            echo "::warning::Potential hardcoded secrets detected - review findings above"
          else
            echo "No hardcoded secrets found"
          fi

      - name: CloudFormation template security lint
        run: |
          pip install cfn-lint
          cfn-lint template.yaml || true
          cfn-lint infrastructure/aws/*.yaml || true
          cfn-lint infrastructure/monitoring/*.yaml || true

  # ================================================================
  # STAGE 3: Build & Validate
  # ================================================================
  build:
    name: Build Lambda Functions
    runs-on: ubuntu-latest
    needs: [test, security-scan]

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Setup AWS SAM CLI
        uses: aws-actions/setup-sam@v2
        with:
          use-installer: true

      - name: Setup pnpm
        uses: pnpm/action-setup@v4

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Install esbuild globally
        run: npm install -g esbuild

      - name: SAM Build
        run: sam build --parallel
        env:
          SAM_CLI_TELEMETRY: 0

      - name: SAM Validate
        run: |
          pip install cfn-lint
          # sam validate runs cfn-lint internally; suppress known SAM false positives
          # E3004: circular dependency between API Gateway and Lambda (SAM resolves this)
          # W8001: unused conditions reserved for future use
          # W1028: !If false-branch unreachable (false-positive for IsProduction conditional creds)
          sam validate 2>&1 || true
          cfn-lint template.yaml -i E3004 W8001 W1028

      - name: Upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: sam-build
          path: .aws-sam/build/
          retention-days: 7

  # ================================================================
  # STAGE 4: Deploy to Staging
  # ================================================================
  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    needs: build
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging'
    environment:
      name: staging
      url: https://staging-api.lyniafinance.com

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Setup AWS SAM CLI
        uses: aws-actions/setup-sam@v2
        with:
          use-installer: true

      - name: Setup pnpm
        uses: pnpm/action-setup@v4

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Install esbuild globally
        run: npm install -g esbuild

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v6
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_REGION }}

      - name: SAM Build
        run: sam build --config-env staging --parallel
        env:
          SAM_CLI_TELEMETRY: 0

      - name: SAM Validate
        run: |
          pip install cfn-lint
          # sam validate runs cfn-lint internally; suppress known SAM false positives
          # E3004: circular dependency between API Gateway and Lambda (SAM resolves this)
          # W8001: unused conditions reserved for future use
          # W1028: !If false-branch unreachable (false-positive for IsProduction conditional creds)
          sam validate 2>&1 || true
          cfn-lint template.yaml -i E3004 W8001 W1028

      - name: Validate database migrations
        run: |
          echo "Checking for pending database migrations..."
          if ls database/migrations/*.sql 1>/dev/null 2>&1; then
            echo "Migrations found - verifying backwards compatibility"
            for f in database/migrations/*.sql; do
              if grep -qiE "DROP (TABLE|COLUMN|INDEX)" "$f" 2>/dev/null; then
                echo "::warning::Destructive migration detected in $f - ensure rollback plan exists"
              fi
            done
          fi
          echo "Migration validation complete"

      - name: Clean up failed stack if in ROLLBACK_COMPLETE
        run: |
          STACK_STATUS=$(aws cloudformation describe-stacks \
            --stack-name lynia-finance-staging \
            --query 'Stacks[0].StackStatus' \
            --output text 2>/dev/null || echo "DOES_NOT_EXIST")
          if [ "$STACK_STATUS" = "ROLLBACK_COMPLETE" ]; then
            echo "Stack is in ROLLBACK_COMPLETE state. Deleting before redeploy..."
            aws cloudformation delete-stack --stack-name lynia-finance-staging
            aws cloudformation wait stack-delete-complete --stack-name lynia-finance-staging
            echo "Stack deleted successfully."
          else
            echo "Stack status: $STACK_STATUS - no cleanup needed."
          fi

      - name: Resolve VPC and Cognito configuration
        id: resolve-staging-infra
        run: |
          VPC_ENABLED="false"
          SUBNET1="subnet-placeholder"
          SUBNET2="subnet-placeholder"
          LAMBDA_SG="sg-00000000"
          if aws cloudformation describe-stacks --stack-name staging-lynia-vpc --region ${{ env.AWS_REGION }} &>/dev/null; then
            SUBNET1=$(aws cloudformation describe-stacks --stack-name staging-lynia-vpc --query "Stacks[0].Outputs[?OutputKey=='PrivateSubnet1Id'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "subnet-placeholder")
            SUBNET2=$(aws cloudformation describe-stacks --stack-name staging-lynia-vpc --query "Stacks[0].Outputs[?OutputKey=='PrivateSubnet2Id'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "subnet-placeholder")
            LAMBDA_SG=$(aws cloudformation describe-stacks --stack-name staging-lynia-vpc --query "Stacks[0].Outputs[?OutputKey=='LambdaSecurityGroupId'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "sg-00000000")
            if [ -n "$SUBNET1" ] && [ "$SUBNET1" != "None" ] && [ "$SUBNET1" != "subnet-placeholder" ]; then VPC_ENABLED="true"; fi
          fi
          echo "vpc_enabled=$VPC_ENABLED" >> $GITHUB_OUTPUT
          echo "subnet1=$SUBNET1" >> $GITHUB_OUTPUT
          echo "subnet2=$SUBNET2" >> $GITHUB_OUTPUT
          echo "lambda_sg=$LAMBDA_SG" >> $GITHUB_OUTPUT
          COGNITO_ARN="arn:aws:cognito-idp:us-east-1:000000000000:userpool/placeholder"
          if aws cloudformation describe-stacks --stack-name staging-lynia-cognito --region ${{ env.AWS_REGION }} &>/dev/null; then
            RESOLVED_ARN=$(aws cloudformation describe-stacks --stack-name staging-lynia-cognito --query "Stacks[0].Outputs[?OutputKey=='UserPoolArn'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "")
            if [ -n "$RESOLVED_ARN" ] && [ "$RESOLVED_ARN" != "None" ]; then COGNITO_ARN="$RESOLVED_ARN"; fi
          fi
          echo "cognito_arn=$COGNITO_ARN" >> $GITHUB_OUTPUT
          # Resolve inter-service API URLs from existing stack (empty for fresh deploys)
          SCORING_API_URL=""
          WHATSAPP_API_URL=""
          KYC_API_URL=""
          if aws cloudformation describe-stacks --stack-name lynia-finance-staging --region ${{ env.AWS_REGION }} &>/dev/null; then
            API_BASE=$(aws cloudformation describe-stacks --stack-name lynia-finance-staging --query "Stacks[0].Outputs[?OutputKey=='ApiGatewayUrl'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "")
            if [ -n "$API_BASE" ] && [ "$API_BASE" != "None" ]; then
              SCORING_API_URL="${API_BASE}scoring/calculate"
              WHATSAPP_API_URL="${API_BASE}whatsapp/send"
              KYC_API_URL="${API_BASE}"
            fi
          fi
          echo "scoring_api_url=$SCORING_API_URL" >> $GITHUB_OUTPUT
          echo "whatsapp_api_url=$WHATSAPP_API_URL" >> $GITHUB_OUTPUT
          echo "kyc_api_url=$KYC_API_URL" >> $GITHUB_OUTPUT

      - name: Check WhatsApp Secrets Manager secret is configured
        id: check-wa-staging
        run: |
          # Verify the WhatsApp credentials are available in AWS Secrets Manager
          # The Lambda resolves them at runtime: lynia-whatsapp-business-credentials
          SECRET_NAME="lynia-whatsapp-business-credentials"
          if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" &>/dev/null; then
            echo "WhatsApp Secrets Manager secret found - bot should be reachable after deploy"
            echo "wa_wired=true" >> $GITHUB_OUTPUT
          else
            echo "::warning::WhatsApp Secrets Manager secret not found: $SECRET_NAME"
            echo "wa_wired=false" >> $GITHUB_OUTPUT
          fi

      - name: SAM Deploy to Staging
        run: |
          # Shell-level fallbacks for secrets (GitHub Actions || operator unreliable for empty secrets)
          WA_PHONE="${WHATSAPP_PHONE_ID:-placeholder}"
          WA_TOKEN="${WHATSAPP_TOKEN:-placeholder}"
          WA_WEBHOOK="${WEBHOOK_TOKEN:-placeholder}"
          WA_BIZ_ACCT="${WHATSAPP_BUSINESS_ACCOUNT_ID:-placeholder}"
          WA_EXPECTED_NUMBER="${WHATSAPP_EXPECTED_DISPLAY_NUMBER:-}"
          META_SECRET="${META_APP_SECRET:-placeholder}"
          DIDIT_KEY="${DIDIT_API_KEY:-placeholder}"
          DIDIT_WH_SECRET="${DIDIT_WEBHOOK_SECRET:-placeholder}"
          ECO_MID="${ECOCASH_MERCHANT_ID:-placeholder}"
          ECO_KEY="${ECOCASH_API_KEY:-placeholder}"
          OM_MID="${ONEMONEY_MERCHANT_ID:-placeholder}"
          OM_KEY="${ONEMONEY_API_KEY:-placeholder}"
          TRUST_KEY="${TRUSTONIC_API_KEY:-placeholder}"
          TRUST_SECRET="${TRUSTONIC_API_SECRET:-placeholder}"
          SMS_KEY="${SMS_API_KEY:-placeholder}"

          # Shell-level fallbacks for inter-service API URLs (empty on fresh stack creation)
          SCORING_URL="${STAGING_SCORING_API_URL:-https://placeholder.execute-api.us-east-1.amazonaws.com/Prod/scoring/calculate}"
          WHATSAPP_URL="${STAGING_WHATSAPP_API_URL:-https://placeholder.execute-api.us-east-1.amazonaws.com/Prod/whatsapp/send}"
          KYC_URL="${STAGING_KYC_API_URL:-https://placeholder.execute-api.us-east-1.amazonaws.com/Prod/}"

          sam deploy \
            --config-env staging \
            --no-confirm-changeset \
            --no-fail-on-empty-changeset \
            --parameter-overrides \
              "Environment=staging" \
              "VpcEnabled=${{ steps.resolve-staging-infra.outputs.vpc_enabled }}" \
              "PrivateSubnet1Id=${{ steps.resolve-staging-infra.outputs.subnet1 }}" \
              "PrivateSubnet2Id=${{ steps.resolve-staging-infra.outputs.subnet2 }}" \
              "LambdaSecurityGroupId=${{ steps.resolve-staging-infra.outputs.lambda_sg }}" \
              "CognitoUserPoolArn=${{ steps.resolve-staging-infra.outputs.cognito_arn }}" \
              "ScoringApiUrl=${SCORING_URL}" \
              "WhatsAppApiUrl=${WHATSAPP_URL}" \
              "KycApiUrl=${KYC_URL}" \
              "WhatsAppExpectedDisplayNumber=${WA_EXPECTED_NUMBER}" \
              "DiditApiKey=${DIDIT_KEY}" \
              "DiditWebhookSecret=${DIDIT_WH_SECRET}" \
              "EcocashMerchantId=${ECO_MID}" \
              "EcocashApiKey=${ECO_KEY}" \
              "OnemoneyMerchantId=${OM_MID}" \
              "OnemoneyApiKey=${OM_KEY}" \
              "TrustonicApiKey=${TRUST_KEY}" \
              "TrustonicApiSecret=${TRUST_SECRET}" \
              "SmsProvider=twilio" \
              "SmsApiKey=${SMS_KEY}"
        env:
          SAM_CLI_TELEMETRY: 0
          STAGING_SCORING_API_URL: ${{ steps.resolve-staging-infra.outputs.scoring_api_url }}
          STAGING_WHATSAPP_API_URL: ${{ steps.resolve-staging-infra.outputs.whatsapp_api_url }}
          STAGING_KYC_API_URL: ${{ steps.resolve-staging-infra.outputs.kyc_api_url }}
          WHATSAPP_PHONE_ID: ${{ secrets.STAGING_WHATSAPP_PHONE_ID }}
          WHATSAPP_TOKEN: ${{ secrets.STAGING_WHATSAPP_TOKEN }}
          WEBHOOK_TOKEN: ${{ secrets.STAGING_WEBHOOK_TOKEN }}
          WHATSAPP_BUSINESS_ACCOUNT_ID: ${{ secrets.STAGING_WHATSAPP_BUSINESS_ACCOUNT_ID }}
          WHATSAPP_EXPECTED_DISPLAY_NUMBER: ${{ secrets.STAGING_WHATSAPP_EXPECTED_DISPLAY_NUMBER }}
          META_APP_SECRET: ${{ secrets.STAGING_META_APP_SECRET }}
          DIDIT_API_KEY: ${{ secrets.STAGING_DIDIT_API_KEY }}
          DIDIT_WEBHOOK_SECRET: ${{ secrets.STAGING_DIDIT_WEBHOOK_SECRET }}
          ECOCASH_MERCHANT_ID: ${{ secrets.STAGING_ECOCASH_MERCHANT_ID }}
          ECOCASH_API_KEY: ${{ secrets.STAGING_ECOCASH_API_KEY }}
          ONEMONEY_MERCHANT_ID: ${{ secrets.STAGING_ONEMONEY_MERCHANT_ID }}
          ONEMONEY_API_KEY: ${{ secrets.STAGING_ONEMONEY_API_KEY }}
          TRUSTONIC_API_KEY: ${{ secrets.STAGING_TRUSTONIC_API_KEY }}
          TRUSTONIC_API_SECRET: ${{ secrets.STAGING_TRUSTONIC_API_SECRET }}
          SMS_API_KEY: ${{ secrets.STAGING_SMS_API_KEY }}

      - name: Get API Gateway URL
        id: get-url
        run: |
          API_URL=$(aws cloudformation describe-stacks \
            --stack-name lynia-finance-staging \
            --query 'Stacks[0].Outputs[?OutputKey==`ApiGatewayUrl`].OutputValue' \
            --output text)
          echo "api_url=$API_URL" >> $GITHUB_OUTPUT
          echo "API Gateway URL: $API_URL"

      - name: Staging smoke tests
        run: |
          API_URL="${{ steps.get-url.outputs.api_url }}"
          echo "Running staging smoke tests against $API_URL"

          FAILED=0

          # Health endpoint
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$API_URL/health" || echo "000")
          if [[ "$HTTP_CODE" =~ ^(200|403|401)$ ]]; then
            echo "Health endpoint: OK ($HTTP_CODE)"
          else
            echo "::warning::Health endpoint returned $HTTP_CODE"
            FAILED=1
          fi

          # Scoring service
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$API_URL/scoring/health" || echo "000")
          echo "Scoring service: $HTTP_CODE"

          # Payment service
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$API_URL/payments/health" || echo "000")
          echo "Payment service: $HTTP_CODE"

          # WhatsApp deep health check - proves the Lambda is bound to the
          # configured Meta Phone Number ID and (when WhatsAppExpectedDisplayNumber
          # is set) that Meta reports the expected business number. This is
          # the only thing that actually verifies the WhatsApp bot will reply
          # to incoming customer messages. Only runs when WA secrets were wired.
          if [ "${{ steps.check-wa-staging.outputs.wa_wired }}" = "true" ]; then
            WA_RESPONSE=$(curl -s --max-time 15 "$API_URL/whatsapp/health?deep=true" || echo '{}')
            WA_CONNECTED=$(echo "$WA_RESPONSE" | jq -r '.connection.connected // false')
            WA_DISPLAY=$(echo "$WA_RESPONSE" | jq -r '.connection.displayPhoneNumber // empty')
            WA_NAME=$(echo "$WA_RESPONSE" | jq -r '.connection.verifiedName // empty')
            if [ "$WA_CONNECTED" = "true" ]; then
              echo "WhatsApp bot connected: $WA_DISPLAY ($WA_NAME)"
            else
              WA_ERROR=$(echo "$WA_RESPONSE" | jq -r '.connection.error // "unknown"')
              echo "::warning::WhatsApp bot NOT connected in staging: $WA_ERROR"
              FAILED=1
            fi
          else
            echo "::notice::Skipping WhatsApp deep health check (secrets not wired)"
          fi

          if [ "$FAILED" -eq 1 ]; then
            echo "::warning::Some smoke tests failed - review before promoting to production"
          fi

      - name: Comment PR with deployment info
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: `### Deployed to Staging\n\n**API URL**: ${{ steps.get-url.outputs.api_url }}\n\n**Environment**: staging\n**Commit**: ${context.sha.substring(0, 7)}\n\nTest your changes at the URL above.`
            })

  # ================================================================
  # STAGE 5: Deploy to Production (manual approval required)
  # ================================================================
  deploy-production:
    name: Deploy to Production
    runs-on: ubuntu-latest
    needs: build
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'production'
    environment:
      name: production
      url: https://api.lyniafinance.com

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Setup AWS SAM CLI
        uses: aws-actions/setup-sam@v2
        with:
          use-installer: true

      - name: Setup pnpm
        uses: pnpm/action-setup@v4

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Install esbuild globally
        run: npm install -g esbuild

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v6
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Verify staging deployment exists
        run: |
          echo "Verifying staging stack exists and is healthy..."
          STAGING_STATUS=$(aws cloudformation describe-stacks \
            --stack-name lynia-finance-staging \
            --query 'Stacks[0].StackStatus' \
            --output text 2>/dev/null || echo "DOES_NOT_EXIST")

          if [[ "$STAGING_STATUS" != *"COMPLETE"* ]] || [[ "$STAGING_STATUS" == "ROLLBACK_COMPLETE" ]]; then
            echo "::error::Staging stack status is $STAGING_STATUS. Deploy to staging first."
            exit 1
          fi
          echo "Staging stack status: $STAGING_STATUS"

      - name: Validate database migrations
        run: |
          echo "Checking for pending database migrations..."
          if ls database/migrations/*.sql 1>/dev/null 2>&1; then
            for f in database/migrations/*.sql; do
              if grep -qiE "DROP (TABLE|COLUMN|INDEX)" "$f" 2>/dev/null; then
                echo "::error::Destructive migration detected in $f - manual review required"
              fi
            done
          fi
          echo "Migration validation complete"

      - name: Record pre-deployment state
        id: pre-deploy
        run: |
          echo "Recording pre-deployment state for rollback reference..."
          PREV_VERSION=$(aws cloudformation describe-stacks \
            --stack-name lynia-finance-prod \
            --query 'Stacks[0].Outputs[?OutputKey==`ApiGatewayUrl`].OutputValue' \
            --output text 2>/dev/null || echo "none")
          PREV_TEMPLATE=$(aws cloudformation get-template \
            --stack-name lynia-finance-prod \
            --template-stage Processed 2>/dev/null | jq -r '.TemplateBody' | md5sum | cut -d' ' -f1 || echo "none")
          echo "prev_api_url=$PREV_VERSION" >> $GITHUB_OUTPUT
          echo "prev_template_hash=$PREV_TEMPLATE" >> $GITHUB_OUTPUT
          echo "deploy_start=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_OUTPUT

      - name: SAM Build
        run: sam build --config-env production --parallel
        env:
          SAM_CLI_TELEMETRY: 0

      - name: SAM Validate
        run: |
          pip install cfn-lint
          # sam validate runs cfn-lint internally; suppress known SAM false positives
          # E3004: circular dependency between API Gateway and Lambda (SAM resolves this)
          # W8001: unused conditions reserved for future use
          # W1028: !If false-branch unreachable (false-positive for IsProduction conditional creds)
          sam validate 2>&1 || true
          cfn-lint template.yaml -i E3004 W8001 W1028

      - name: Resolve VPC and Cognito configuration
        id: resolve-prod-infra
        run: |
          VPC_ENABLED="false"
          SUBNET1="subnet-placeholder"
          SUBNET2="subnet-placeholder"
          LAMBDA_SG="sg-00000000"
          if aws cloudformation describe-stacks --stack-name production-lynia-vpc --region ${{ env.AWS_REGION }} &>/dev/null; then
            SUBNET1=$(aws cloudformation describe-stacks --stack-name production-lynia-vpc --query "Stacks[0].Outputs[?OutputKey=='PrivateSubnet1Id'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "subnet-placeholder")
            SUBNET2=$(aws cloudformation describe-stacks --stack-name production-lynia-vpc --query "Stacks[0].Outputs[?OutputKey=='PrivateSubnet2Id'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "subnet-placeholder")
            LAMBDA_SG=$(aws cloudformation describe-stacks --stack-name production-lynia-vpc --query "Stacks[0].Outputs[?OutputKey=='LambdaSecurityGroupId'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "sg-00000000")
            if [ -n "$SUBNET1" ] && [ "$SUBNET1" != "None" ] && [ "$SUBNET1" != "subnet-placeholder" ]; then VPC_ENABLED="true"; fi
          fi
          echo "vpc_enabled=$VPC_ENABLED" >> $GITHUB_OUTPUT
          echo "subnet1=$SUBNET1" >> $GITHUB_OUTPUT
          echo "subnet2=$SUBNET2" >> $GITHUB_OUTPUT
          echo "lambda_sg=$LAMBDA_SG" >> $GITHUB_OUTPUT
          COGNITO_ARN="arn:aws:cognito-idp:us-east-1:000000000000:userpool/placeholder"
          if aws cloudformation describe-stacks --stack-name production-lynia-cognito --region ${{ env.AWS_REGION }} &>/dev/null; then
            RESOLVED_ARN=$(aws cloudformation describe-stacks --stack-name production-lynia-cognito --query "Stacks[0].Outputs[?OutputKey=='UserPoolArn'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "")
            if [ -n "$RESOLVED_ARN" ] && [ "$RESOLVED_ARN" != "None" ]; then COGNITO_ARN="$RESOLVED_ARN"; fi
          fi
          echo "cognito_arn=$COGNITO_ARN" >> $GITHUB_OUTPUT
          # Resolve inter-service API URLs from existing stack
          SCORING_API_URL=""
          WHATSAPP_API_URL=""
          KYC_API_URL=""
          if aws cloudformation describe-stacks --stack-name lynia-finance-prod --region ${{ env.AWS_REGION }} &>/dev/null; then
            API_BASE=$(aws cloudformation describe-stacks --stack-name lynia-finance-prod --query "Stacks[0].Outputs[?OutputKey=='ApiGatewayUrl'].OutputValue" --output text --region ${{ env.AWS_REGION }} 2>/dev/null || echo "")
            if [ -n "$API_BASE" ] && [ "$API_BASE" != "None" ]; then
              SCORING_API_URL="${API_BASE}scoring/calculate"
              WHATSAPP_API_URL="${API_BASE}whatsapp/send"
              KYC_API_URL="${API_BASE}"
            fi
          fi
          echo "scoring_api_url=$SCORING_API_URL" >> $GITHUB_OUTPUT
          echo "whatsapp_api_url=$WHATSAPP_API_URL" >> $GITHUB_OUTPUT
          echo "kyc_api_url=$KYC_API_URL" >> $GITHUB_OUTPUT

      - name: Check WhatsApp Secrets Manager secret is configured
        id: check-wa-prod
        run: |
          # Hard gate: a production deploy without the WhatsApp Business
          # credentials would silently ship a broken bot to real customers.
          # The landing page WhatsApp button points at +263 71 925 2094 and
          # clicks must reach the bot. Credentials are now stored in AWS
          # Secrets Manager: lynia-whatsapp-business-credentials
          SECRET_NAME="lynia-whatsapp-business-credentials"
          if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" &>/dev/null; then
            echo "WhatsApp Secrets Manager secret found - bot will be live after deploy"
            echo "wa_wired=true" >> $GITHUB_OUTPUT
          else
            if [ "${FORCE_DEPLOY_WITHOUT_WA}" = "true" ]; then
              echo "::warning::force_deploy_without_wa=true - deploying anyway. Secret not found: $SECRET_NAME"
              echo "wa_wired=false" >> $GITHUB_OUTPUT
            else
              echo "::error::Cannot deploy to production: missing WhatsApp secret in AWS Secrets Manager: $SECRET_NAME"
              echo "::error::Create it in AWS Secrets Manager, or re-run with force_deploy_without_wa=true"
              exit 1
            fi
          fi
        env:
          FORCE_DEPLOY_WITHOUT_WA: ${{ github.event.inputs.force_deploy_without_wa }}

      - name: SAM Deploy to Production
        run: |
          # Shell-level fallbacks for secrets (GitHub Actions || operator unreliable for empty secrets)
          WA_PHONE="${WHATSAPP_PHONE_ID:-placeholder}"
          WA_TOKEN="${WHATSAPP_TOKEN:-placeholder}"
          WA_WEBHOOK="${WEBHOOK_TOKEN:-placeholder}"
          WA_BIZ_ACCT="${WHATSAPP_BUSINESS_ACCOUNT_ID:-placeholder}"
          # Hard-coded to the live Lynia Business number for production.
          # This ensures the health check always verifies the correct number
          # is bound to the backend. See
          # docs/external-integrations/WHATSAPP-BUSINESS-NUMBER-VERIFICATION.md
          WA_EXPECTED_NUMBER="+263 71 925 2094"
          META_SECRET="${META_APP_SECRET:-placeholder}"
          DIDIT_KEY="${DIDIT_API_KEY:-placeholder}"
          DIDIT_WH_SECRET="${DIDIT_WEBHOOK_SECRET:-placeholder}"
          ECO_MID="${ECOCASH_MERCHANT_ID:-placeholder}"
          ECO_KEY="${ECOCASH_API_KEY:-placeholder}"
          OM_MID="${ONEMONEY_MERCHANT_ID:-placeholder}"
          OM_KEY="${ONEMONEY_API_KEY:-placeholder}"
          TRUST_KEY="${TRUSTONIC_API_KEY:-placeholder}"
          TRUST_SECRET="${TRUSTONIC_API_SECRET:-placeholder}"
          SMS_KEY="${SMS_API_KEY:-placeholder}"

          # Shell-level fallbacks for inter-service API URLs (empty on fresh stack creation)
          SCORING_URL="${PROD_SCORING_API_URL:-https://placeholder.execute-api.us-east-1.amazonaws.com/Prod/scoring/calculate}"
          WHATSAPP_URL="${PROD_WHATSAPP_API_URL:-https://placeholder.execute-api.us-east-1.amazonaws.com/Prod/whatsapp/send}"
          KYC_URL="${PROD_KYC_API_URL:-https://placeholder.execute-api.us-east-1.amazonaws.com/Prod/}"

          sam deploy \
            --config-env production \
            --no-confirm-changeset \
            --no-fail-on-empty-changeset \
            --on-failure ROLLBACK \
            --parameter-overrides \
              "Environment=production" \
              "VpcEnabled=${{ steps.resolve-prod-infra.outputs.vpc_enabled }}" \
              "PrivateSubnet1Id=${{ steps.resolve-prod-infra.outputs.subnet1 }}" \
              "PrivateSubnet2Id=${{ steps.resolve-prod-infra.outputs.subnet2 }}" \
              "LambdaSecurityGroupId=${{ steps.resolve-prod-infra.outputs.lambda_sg }}" \
              "CognitoUserPoolArn=${{ steps.resolve-prod-infra.outputs.cognito_arn }}" \
              "ScoringApiUrl=${SCORING_URL}" \
              "WhatsAppApiUrl=${WHATSAPP_URL}" \
              "KycApiUrl=${KYC_URL}" \
              "WhatsAppExpectedDisplayNumber=${WA_EXPECTED_NUMBER}" \
              "DiditApiKey=${DIDIT_KEY}" \
              "DiditWebhookSecret=${DIDIT_WH_SECRET}" \
              "EcocashMerchantId=${ECO_MID}" \
              "EcocashApiKey=${ECO_KEY}" \
              "OnemoneyMerchantId=${OM_MID}" \
              "OnemoneyApiKey=${OM_KEY}" \
              "TrustonicApiKey=${TRUST_KEY}" \
              "TrustonicApiSecret=${TRUST_SECRET}" \
              "SmsProvider=twilio" \
              "SmsApiKey=${SMS_KEY}"
        env:
          SAM_CLI_TELEMETRY: 0
          PROD_SCORING_API_URL: ${{ steps.resolve-prod-infra.outputs.scoring_api_url }}
          PROD_WHATSAPP_API_URL: ${{ steps.resolve-prod-infra.outputs.whatsapp_api_url }}
          PROD_KYC_API_URL: ${{ steps.resolve-prod-infra.outputs.kyc_api_url }}
          WHATSAPP_PHONE_ID: ${{ secrets.PRODUCTION_WHATSAPP_PHONE_ID }}
          WHATSAPP_TOKEN: ${{ secrets.PRODUCTION_WHATSAPP_TOKEN }}
          WEBHOOK_TOKEN: ${{ secrets.PRODUCTION_WEBHOOK_TOKEN }}
          WHATSAPP_BUSINESS_ACCOUNT_ID: ${{ secrets.PRODUCTION_WHATSAPP_BUSINESS_ACCOUNT_ID }}
          # Note: WHATSAPP_EXPECTED_DISPLAY_NUMBER is hard-coded in the shell script above
          # to ensure the health check always verifies the correct number is bound.
          # Do not override it with a GitHub secret.
          META_APP_SECRET: ${{ secrets.PRODUCTION_META_APP_SECRET }}
          DIDIT_API_KEY: ${{ secrets.PRODUCTION_DIDIT_API_KEY }}
          DIDIT_WEBHOOK_SECRET: ${{ secrets.PRODUCTION_DIDIT_WEBHOOK_SECRET }}
          ECOCASH_MERCHANT_ID: ${{ secrets.PRODUCTION_ECOCASH_MERCHANT_ID }}
          ECOCASH_API_KEY: ${{ secrets.PRODUCTION_ECOCASH_API_KEY }}
          ONEMONEY_MERCHANT_ID: ${{ secrets.PRODUCTION_ONEMONEY_MERCHANT_ID }}
          ONEMONEY_API_KEY: ${{ secrets.PRODUCTION_ONEMONEY_API_KEY }}
          TRUSTONIC_API_KEY: ${{ secrets.PRODUCTION_TRUSTONIC_API_KEY }}
          TRUSTONIC_API_SECRET: ${{ secrets.PRODUCTION_TRUSTONIC_API_SECRET }}
          SMS_API_KEY: ${{ secrets.PRODUCTION_SMS_API_KEY }}

      - name: Get Production API Gateway URL
        id: get-prod-url
        run: |
          API_URL=$(aws cloudformation describe-stacks \
            --stack-name lynia-finance-prod \
            --query 'Stacks[0].Outputs[?OutputKey==`ApiGatewayUrl`].OutputValue' \
            --output text)
          echo "api_url=$API_URL" >> $GITHUB_OUTPUT
          echo "Production API Gateway URL: $API_URL"

      - name: Production smoke tests
        id: smoke-tests
        run: |
          API_URL="${{ steps.get-prod-url.outputs.api_url }}"
          echo "Running production smoke tests..."
          FAILED=0

          for endpoint in health scoring/health payments/health; do
            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
              "$API_URL/$endpoint" || echo "000")
            if [[ "$HTTP_CODE" =~ ^(200|403|401)$ ]]; then
              echo "$endpoint: OK ($HTTP_CODE)"
            else
              echo "::warning::$endpoint returned $HTTP_CODE"
              FAILED=1
            fi
          done

          # WhatsApp deep health check - confirms the production Lambda is
          # bound to +263 71 925 2094 (WhatsAppExpectedDisplayNumber) and
          # that Meta reports the WhatsApp Business number as connected.
          # This is the single authoritative signal that the WhatsApp bot
          # will actually respond to customer messages post-deploy.
          if [ "${{ steps.check-wa-prod.outputs.wa_wired }}" = "true" ]; then
            echo "--- WhatsApp Business connection check ---"
            WA_RESPONSE=$(curl -s --max-time 15 "$API_URL/whatsapp/health?deep=true" || echo '{}')
            WA_CONNECTED=$(echo "$WA_RESPONSE" | jq -r '.connection.connected // false')
            WA_MATCHES=$(echo "$WA_RESPONSE" | jq -r '.connection.matchesExpected // null')
            WA_DISPLAY=$(echo "$WA_RESPONSE" | jq -r '.connection.displayPhoneNumber // empty')
            WA_NAME=$(echo "$WA_RESPONSE" | jq -r '.connection.verifiedName // empty')
            WA_QUALITY=$(echo "$WA_RESPONSE" | jq -r '.connection.qualityRating // empty')
            if [ "$WA_CONNECTED" = "true" ]; then
              echo "whatsapp: connected to $WA_DISPLAY ($WA_NAME, quality=$WA_QUALITY)"
              if [ "$WA_MATCHES" = "false" ]; then
                echo "::error::WhatsApp Phone Number ID resolves to the wrong business number!"
                FAILED=1
              fi
            else
              WA_ERROR=$(echo "$WA_RESPONSE" | jq -r '.connection.error // "unknown"')
              echo "::error::WhatsApp bot NOT connected: $WA_ERROR"
              FAILED=1
            fi
          else
            echo "::warning::Skipping WhatsApp deep health check (deploy proceeded without WA secrets)"
          fi

          echo "failed=$FAILED" >> $GITHUB_OUTPUT

      - name: Monitor error rate post-deploy (2 min window)
        if: steps.smoke-tests.outputs.failed != '1'
        run: |
          echo "Monitoring error rates for 2 minutes post-deployment..."
          sleep 120

          # Check CloudWatch for elevated errors
          END_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
          START_TIME="${{ steps.pre-deploy.outputs.deploy_start }}"

          ERRORS=$(aws cloudwatch get-metric-statistics \
            --namespace AWS/Lambda \
            --metric-name Errors \
            --start-time "$START_TIME" \
            --end-time "$END_TIME" \
            --period 60 \
            --statistics Sum \
            --query 'Datapoints[].Sum' \
            --output text 2>/dev/null || echo "0")

          echo "Errors since deployment: $ERRORS"
          if [ -n "$ERRORS" ] && [ "$ERRORS" != "None" ] && [ "$ERRORS" != "0" ]; then
            echo "::warning::Errors detected post-deployment - monitor closely"
          fi

      - name: Create GitHub Release
        if: success()
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: v${{ github.run_number }}
          release_name: Production Release v${{ github.run_number }}
          body: |
            **Deployed to Production**

            API URL: ${{ steps.get-prod-url.outputs.api_url }}
            Commit: ${{ github.sha }}
            Deployed: ${{ steps.pre-deploy.outputs.deploy_start }}

            ### Changes
            See commit history for details.
          draft: false
          prerelease: false

  # ================================================================
  # STAGE 6: Notifications
  # ================================================================
  notify:
    name: Deployment Notifications
    runs-on: ubuntu-latest
    needs: [test, security-scan, build, deploy-staging, deploy-production]
    if: always()

    steps:
      - name: Determine deployment result
        id: result
        run: |
          if [ "${{ needs.deploy-production.result }}" == "success" ]; then
            echo "status=Production deployment successful" >> $GITHUB_OUTPUT
            echo "env=production" >> $GITHUB_OUTPUT
            echo "color=good" >> $GITHUB_OUTPUT
          elif [ "${{ needs.deploy-staging.result }}" == "success" ]; then
            echo "status=Staging deployment successful" >> $GITHUB_OUTPUT
            echo "env=staging" >> $GITHUB_OUTPUT
            echo "color=good" >> $GITHUB_OUTPUT
          elif [ "${{ needs.deploy-production.result }}" == "failure" ]; then
            echo "status=PRODUCTION DEPLOYMENT FAILED" >> $GITHUB_OUTPUT
            echo "env=production" >> $GITHUB_OUTPUT
            echo "color=danger" >> $GITHUB_OUTPUT
          elif [ "${{ needs.deploy-staging.result }}" == "failure" ]; then
            echo "status=Staging deployment failed" >> $GITHUB_OUTPUT
            echo "env=staging" >> $GITHUB_OUTPUT
            echo "color=warning" >> $GITHUB_OUTPUT
          elif [ "${{ needs.test.result }}" == "failure" ]; then
            echo "status=Tests failed - deployment blocked" >> $GITHUB_OUTPUT
            echo "env=none" >> $GITHUB_OUTPUT
            echo "color=danger" >> $GITHUB_OUTPUT
          elif [ "${{ needs.security-scan.result }}" == "failure" ]; then
            echo "status=Security scan failed - deployment blocked" >> $GITHUB_OUTPUT
            echo "env=none" >> $GITHUB_OUTPUT
            echo "color=danger" >> $GITHUB_OUTPUT
          else
            echo "status=Deployment skipped" >> $GITHUB_OUTPUT
            echo "env=none" >> $GITHUB_OUTPUT
            echo "color=warning" >> $GITHUB_OUTPUT
          fi

      - name: Send Slack notification
        if: vars.SLACK_WEBHOOK_URL != ''
        run: |
          curl -s -X POST "${{ vars.SLACK_WEBHOOK_URL }}" \
            -H 'Content-type: application/json' \
            -d '{
              "attachments": [{
                "color": "${{ steps.result.outputs.color }}",
                "title": "Lynia Finance Deployment",
                "text": "${{ steps.result.outputs.status }}",
                "fields": [
                  {"title": "Environment", "value": "${{ steps.result.outputs.env }}", "short": true},
                  {"title": "Branch", "value": "${{ github.ref_name }}", "short": true},
                  {"title": "Commit", "value": "${{ github.sha }}", "short": true},
                  {"title": "Actor", "value": "${{ github.actor }}", "short": true}
                ],
                "footer": "GitHub Actions",
                "ts": '$( date +%s )'
              }]
            }' || true

      - name: Send email notification for production failures
        if: |
          needs.deploy-production.result == 'failure' &&
          vars.ALERT_EMAIL != ''
        run: |
          echo "PRODUCTION DEPLOYMENT FAILED"
          echo "Commit: ${{ github.sha }}"
          echo "Actor: ${{ github.actor }}"
          echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          echo ""
          echo "Action required: Check deployment logs and consider rollback."

      - name: Post summary
        run: |
          echo "## Deployment Summary" >> $GITHUB_STEP_SUMMARY
          echo "| Stage | Result |" >> $GITHUB_STEP_SUMMARY
          echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
          echo "| Tests | ${{ needs.test.result }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Security Scan | ${{ needs.security-scan.result }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Build | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Staging Deploy | ${{ needs.deploy-staging.result }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Production Deploy | ${{ needs.deploy-production.result }} |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Status**: ${{ steps.result.outputs.status }}" >> $GITHUB_STEP_SUMMARY