cd.yaml

name: CD

on:
  push:
    tags:
      - 'v*'

permissions: {}

# Shared with the CI merge-queue deploy-prod job (ci.yaml) so a tag deploy
# and a merge-queue deploy can never run against the prod cluster at once.
concurrency:
  group: prod-deploy
  cancel-in-progress: false

jobs:
  deploy-prod:
    name: 🚀 Deploy to Production
    runs-on: ubuntu-latest
    environment: prod
    permissions:
      contents: read # checkout repository
      packages: write # push OCI artifacts to GHCR
      id-token: write # mint OIDC token for cosign keyless signing (Fulcio + Rekor)
      attestations: write # write SBOM + SLSA provenance attestations
      # No artifact-metadata:write — the attest steps below set
      # create-storage-record: false, so the storage-record API is never called.
      # #1787 granted it to clear a "no artifacts found" warning, but the grant
      # was necessary-but-not-sufficient (actions/attest-build-provenance#781);
      # see the Attest SBOM step for the full rationale.
    steps:
      - name: 📑 Checkout
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false

      - name: ⚙️ Setup KSail
        # Install the KSail CLI from the release tarball via curl. We use curl
        # rather than brew/setup-ksail-cli because the v7.20.0 release shipped
        # only desktop artifacts (linux CLI missing, 404 on the tarball).
        # v7.25.0 ships complete linux artifacts again and carries the
        # cluster-update fixes; renovate keeps this pin current.
        env:
          # renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?<version>.+)$
          KSAIL_VERSION: "7.61.0"
        run: |
          curl -fsSL "https://github.com/devantler-tech/ksail/releases/download/v${KSAIL_VERSION}/ksail_${KSAIL_VERSION}_linux_amd64.tar.gz" -o /tmp/ksail.tar.gz
          tar -xzf /tmp/ksail.tar.gz -C /tmp
          sudo install /tmp/ksail /usr/local/bin/ksail
          ksail --version

      - name: 🔐 Create SOPS Age key
        env:
          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
        run: |
          mkdir -p ~/.config/sops/age
          umask 077
          echo "${SOPS_AGE_KEY}" > ~/.config/sops/age/keys.txt
          chmod 600 ~/.config/sops/age/keys.txt

      - name: 🔑 Restore kubeconfig
        # ksail cluster update needs a valid kubeconfig to detect the current
        # cluster state. Without it, the component detector returns defaults and
        # triggers spurious changes that all fail. The KUBE_CONFIG secret must be
        # kept up-to-date from the homelab after any cluster recreation.
        # See devantler-tech/ksail#4369 for the upstream design context.
        env:
          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
        run: |
          mkdir -p ~/.kube
          umask 077
          printf '%s' "${KUBE_CONFIG}" > ~/.kube/config
          chmod 600 ~/.kube/config

      - name: 🔑 Restore talosconfig
        # ksail cluster update talks to the Talos API to sync machine config.
        # Without the talosconfig it uses stale/derived certs and fails with
        # "x509: certificate signed by unknown authority (talos)". The
        # TALOS_CONFIG secret must be kept up-to-date from the homelab after any
        # cluster recreation, same as KUBE_CONFIG.
        env:
          TALOS_CONFIG: ${{ secrets.TALOS_CONFIG }}
        run: |
          mkdir -p ~/.talos
          umask 077
          printf '%s' "${TALOS_CONFIG}" > ~/.talos/config
          chmod 600 ~/.talos/config

      - name: 🩺 Verify prod cluster is reachable
        # Fail fast if the restored kubeconfig can't reach the prod API server.
        # When the cluster is unreachable — most commonly because KUBE_CONFIG /
        # TALOS_CONFIG are stale after a cluster rebuild — `ksail cluster update`
        # otherwise only warns, falls back to a "nothing is installed" baseline,
        # proposes a full reinstall, and then fails mid-apply against the live
        # nodes (a confusing, potentially destructive failure mode). Catch it
        # here with an actionable message instead.
        # Upstream: devantler-tech/ksail#4868 (fail fast) and #4869 (baseline).
        # Refresh procedure: docs/dr/runbook.md → "Refresh CI deploy credentials".
        # Pin --context to admin@prod (ksail.prod.yaml's connection.context) so
        # the check targets the same cluster `ksail cluster update` will use,
        # regardless of the restored kubeconfig's current-context.
        run: |
          # A missing admin@prod context means a malformed/incomplete kubeconfig,
          # not an unreachable cluster — surface that case distinctly.
          if ! kubectl config get-contexts -o name | grep -qx 'admin@prod'; then
            echo "::error::Restored kubeconfig has no 'admin@prod' context (ksail.prod.yaml's connection.context). KUBE_CONFIG looks malformed or incomplete — refresh it. See docs/dr/runbook.md → 'Refresh CI deploy credentials'."
            exit 1
          fi
          last_err=""
          for attempt in 1 2 3; do
            # Capture kubectl's own stderr so the failure message is accurate.
            if out=$(kubectl --context admin@prod get --raw='/readyz' --request-timeout=30s 2>&1); then
              echo "✅ Prod API server reachable."
              exit 0
            fi
            last_err="$out"
            echo "Attempt ${attempt}/3: prod API server (context admin@prod) not reachable: ${out}"
            sleep 10
          done
          last_err=$(printf '%s' "$last_err" | tr '\n' ' ')
          echo "::error::Prod cluster (context admin@prod) is unreachable with the restored kubeconfig (last kubectl error: ${last_err}). KUBE_CONFIG (and likely TALOS_CONFIG) are probably stale after a cluster rebuild — refresh both and update the 'prod' environment secrets. See docs/dr/runbook.md → 'Refresh CI deploy credentials'."
          exit 1

      # Publish the GitOps manifests (push → sign → attest → reconcile) BEFORE
      # syncing Talos machine config (the "🔄 Update cluster" step now runs last).
      # Manifest delivery never touches the nodes, so it must not be gated on
      # every control-plane node being reachable: when one node's Talos API was
      # down, `ksail cluster update` ran first, hard-failed the job, and froze
      # ALL app/config delivery to an otherwise-healthy cluster (the apiserver
      # and Flux were fine on the two surviving control-plane nodes).
      - name: 📦 Push manifests to GHCR
        run: ksail --config ksail.prod.yaml workload push
        env:
          GITHUB_ACTOR: ${{ github.actor }}
          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}

      - name: ⚙️ Install cosign
        # sigstore/cosign-installer is the official installer action — it
        # downloads cosign for the requested release and verifies its own
        # Sigstore signature against Fulcio + Rekor (keyless), so there is
        # no per-version SHA256 to track in this repo. Bump cosign by
        # editing COSIGN_VERSION below; Renovate proposes the bump via the
        # custom regex manager in .github/renovate.json.
        env:
          # renovate: datasource=github-releases depName=sigstore/cosign extractVersion=^v(?<version>.+)$
          COSIGN_VERSION: "3.1.1"
        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
        with:
          cosign-release: 'v${{ env.COSIGN_VERSION }}'

      - name: 🔐 Configure GHCR auth (cosign + buildx + syft)
        # Pre-create ~/.docker/config.json instead of `cosign login` /
        # `docker login` so neither emits the "credentials stored
        # unencrypted in '/home/runner/.docker/config.json'" warning every
        # run. The on-disk format is byte-identical to what those CLI
        # commands would have written (base64 of "actor:token") -- the
        # warning is cosmetic noise on an ephemeral runner that's destroyed
        # after the job. Writing the auth here once also lets us drop the
        # per-tool login commands from the steps below: cosign, docker
        # buildx imagetools, and syft (via anchore/sbom-action) all resolve
        # GHCR credentials through the standard OCI keychain, which reads
        # this same file.
        env:
          GITHUB_ACTOR: ${{ github.actor }}
          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
        run: |
          set -euo pipefail
          umask 077
          mkdir -p ~/.docker
          # Build the auth blob in a command substitution so it never
          # reaches stdout.
          AUTH=$(printf '%s:%s' "${GITHUB_ACTOR}" "${GHCR_TOKEN}" | base64 -w0)
          # Register the derived value as a workflow mask so any future
          # accidental log output (e.g. from a `set -x` added during
          # debugging) is redacted -- GitHub's automatic secret masking
          # only matches the raw GHCR_TOKEN literal, not the base64 of
          # "actor:token" derived from it.
          echo "::add-mask::${AUTH}"
          # jq is preinstalled on ubuntu-latest; --arg avoids any shell
          # quoting issue if the token ever contains a JSON-special char.
          jq -n --arg auth "${AUTH}" \
            '{auths: {"ghcr.io": {auth: $auth}}}' \
            > ~/.docker/config.json
          chmod 600 ~/.docker/config.json

      - name: 🖋️ Sign OCI manifest with cosign (keyless)
        # Signs the platform OCI artifact published by `ksail workload push`
        # using Sigstore keyless signing: Fulcio mints a short-lived cert
        # bound to this workflow's OIDC identity and Rekor records the
        # signature in its append-only transparency log. There is no
        # private key on disk -- the security policy lives in the
        # verifier's matchOIDCIdentity regex (planned for a follow-up).
        id: cosign-sign
        env:
          # COSIGN_EXPERIMENTAL was retired in cosign 2.x; keyless is now
          # the default when --key is omitted. GHCR credentials come from
          # ~/.docker/config.json populated by the "Configure GHCR auth"
          # step above; cosign reads them through the OCI keychain.
          COSIGN_YES: "true"
        run: |
          set -euo pipefail
          REF_TAG="ghcr.io/devantler-tech/platform/manifests:latest"
          # Resolve the tag → digest BEFORE signing so that (a) cosign signs
          # exactly the bytes we resolved -- no tag→digest TOCTOU if a
          # concurrent deploy mutates the tag between this resolve and the
          # sign call, (b) cosign stops warning about tag-based references
          # and we stay forward-compatible with the cosign roadmap that
          # removes tag-based signing entirely, and (c) cosign skips its
          # own internal tag→digest lookup -- one fewer registry round-trip.
          # docker buildx imagetools is preinstalled on ubuntu-latest; the
          # cosign-native equivalent (`cosign manifest digest`) was removed
          # in cosign v3.0.
          DIGEST=$(docker buildx imagetools inspect "${REF_TAG}" --format '{{.Manifest.Digest}}')
          REF="ghcr.io/devantler-tech/platform/manifests@${DIGEST}"
          cosign sign --yes --recursive "${REF}"
          echo "digest=${DIGEST}" >> "${GITHUB_OUTPUT}"
          echo "ref=${REF_TAG}" >> "${GITHUB_OUTPUT}"

      - name: 📋 Generate SBOM (CycloneDX)
        # anchore/sbom-action is the official installer wrapper for syft —
        # it manages syft's binary integrity itself, so no per-version
        # SHA256 to track here either. Bump syft by editing SYFT_VERSION
        # below; Renovate proposes the bump via the custom regex manager.
        # Output is CycloneDX JSON, a format actions/attest accepts via sbom-path.
        # `upload-artifact: false` matches today's behaviour (no workflow
        # artifact); the SBOM is attested to the registry in the next step.
        env:
          # renovate: datasource=github-releases depName=anchore/syft extractVersion=^v(?<version>.+)$
          SYFT_VERSION: "1.45.1"
        uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0
        with:
          image: ghcr.io/devantler-tech/platform/manifests@${{ steps.cosign-sign.outputs.digest }}
          format: cyclonedx-json
          output-file: sbom.cdx.json
          syft-version: 'v${{ env.SYFT_VERSION }}'
          upload-artifact: false

      - name: 🪪 Attest SBOM (Sigstore via cosign keyless)
        # Records the SBOM as a Sigstore in-toto attestation against the
        # OCI manifest's digest. The attestation is uploaded to GHCR and
        # its index recorded in Rekor. actions/attest-sbom v4 is now just a
        # deprecated wrapper around actions/attest, so call attest directly
        # and let its sbom-path input auto-select the SBOM predicate type.
        #
        # create-storage-record: false opts out of GitHub's artifact-metadata
        # storage record — a GHCR "linked artifacts" discoverability index, not a
        # supply-chain control. It cannot succeed here: the manifest is pushed by
        # oras/ksail + cosign, not a GitHub-native artifact flow, so the
        # artifact-metadata service has no registered artifact to attach to and
        # warns "no artifacts found" on every deploy. #1787 mis-read this as a
        # missing artifact-metadata:write grant; the grant shipped (v1.34.5) and
        # the warning persisted, because per actions/attest-build-provenance#781
        # the permission is necessary-but-not-sufficient. The cosign signature,
        # SBOM attestation, SLSA provenance, and Rekor entry are all unaffected.
        uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0
        with:
          subject-digest: ${{ steps.cosign-sign.outputs.digest }}
          subject-name: ghcr.io/devantler-tech/platform/manifests
          sbom-path: sbom.cdx.json
          push-to-registry: true
          create-storage-record: false

      - name: 🪪 Attest build provenance (SLSA L3)
        # Generates a SLSA provenance v1.0 statement and attests it against
        # the same digest. Provenance captures the workflow file,
        # invocation parameters, GitHub runner, source repo + SHA, etc.
        # Combined with the cosign signature above and the SBOM
        # attestation, this completes the supply-chain transparency
        # bundle for the platform OCI artifact.
        # create-storage-record: false for the same reason as the SBOM step above.
        uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
        with:
          subject-digest: ${{ steps.cosign-sign.outputs.digest }}
          subject-name: ghcr.io/devantler-tech/platform/manifests
          push-to-registry: true
          create-storage-record: false

      - name: 🔁 Trigger Flux reconciliation
        id: reconcile
        run: ksail --config ksail.prod.yaml workload reconcile
        env:
          GITHUB_ACTOR: ${{ github.actor }}
          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}

      - name: 🩺 Diagnose Flux on failure
        if: failure() && steps.reconcile.conclusion == 'failure'
        env:
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
        run: |
          set +e
          echo "::group::Flux Kustomizations"
          kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide
          echo "::endgroup::"
          echo "::group::Flux HelmReleases"
          kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o wide
          echo "::endgroup::"
          echo "::group::Flux OCIRepositories"
          kubectl get ocirepositories.source.toolkit.fluxcd.io -A -o wide
          echo "::endgroup::"
          echo "::group::Failing Kustomizations describe"
          for k in infrastructure-controllers infrastructure apps; do
            echo "--- $k ---"
            kubectl describe kustomizations.kustomize.toolkit.fluxcd.io "$k" -n flux-system 2>&1 | tail -60
          done
          echo "::endgroup::"
          echo "::group::Flux controller pods"
          kubectl get pods -n flux-system -o wide
          echo "::endgroup::"
          echo "::group::kustomize-controller logs"
          kubectl logs -n flux-system -l app=kustomize-controller --tail=200
          echo "::endgroup::"
          echo "::group::helm-controller logs"
          kubectl logs -n flux-system -l app=helm-controller --tail=200
          echo "::endgroup::"
          echo "::group::source-controller logs"
          kubectl logs -n flux-system -l app=source-controller --tail=100
          echo "::endgroup::"
          echo "::group::Recent events (warnings)"
          kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -50
          echo "::endgroup::"

      - name: 🔄 Update cluster (Talos machine config)
        # Runs LAST, after manifest delivery, so an unreachable control-plane
        # node cannot block GitOps delivery (see the Push step's comment). Talos
        # config sync still runs on every deploy and a genuine node-reachability
        # problem still fails the deploy (red) — it just no longer takes the
        # app/config pipeline down with it. Automatically skipped if an earlier
        # delivery step already failed (no point syncing nodes for a broken
        # release).
        run: ksail --config ksail.prod.yaml cluster update
        env:
          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}