Let the app run on more than one node #115
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| tags: ['v*'] | |
| workflow_dispatch: | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| lint-manifests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install kubeconform | |
| run: | | |
| curl -sSL https://github.com/yannh/kubeconform/releases/latest/download/kubeconform-linux-amd64.tar.gz | tar xz | |
| sudo mv kubeconform /usr/local/bin/ | |
| - name: Install kubectl | |
| # Pinned to the version helm/kind-action installs in the kind jobs, so | |
| # every job in this file renders through the same kustomize (v5.7.1). | |
| # Unpinned, setup-kubectl@v3 resolves `latest` from the frozen legacy | |
| # GCS bucket - kubectl v1.31.0, kustomize 5.4.2 - and the static gates | |
| # then validate output the kind clusters never see. | |
| uses: azure/setup-kubectl@v3 | |
| with: | |
| version: v1.35.0 | |
| - name: Set up Helm | |
| # kubectl kustomize --enable-helm shells out to `helm` to inflate the | |
| # Ganesha chart in k8s/storage/. | |
| # | |
| # PINNED TO THE 3.x LINE ON PURPOSE, and pinned identically at all four | |
| # `Set up Helm` steps in this file. The kustomize built into kubectl | |
| # <= 1.35 (v5.7.1) probes the binary with `helm version -c --short` - a | |
| # shorthand Helm 4 removed - and then refuses any major that is not 3. | |
| # An unpinned `latest` resolved to Helm v4.2.4 and took every job that | |
| # renders k8s/storage/ down with | |
| # error: unknown shorthand flag: 'c' in -c | |
| # Do NOT "fix" this by bumping kubectl past 1.36 instead: kustomize | |
| # v5.8.1 stopped applying the namespace transformer to Helm-inflated | |
| # objects, so the chart's ServiceAccount, Service and StatefulSet would | |
| # render with no metadata.namespace - contradicting the contract stated | |
| # at the top of k8s/storage/kustomization.yaml, and silently deploying | |
| # Ganesha outside its NetworkPolicy for anyone using the documented | |
| # `kustomize | kubectl apply -f -` pipe. | |
| uses: azure/setup-helm@v4 | |
| with: | |
| # renovate: datasource=github-releases depName=helm/helm | |
| version: v3.21.4 | |
| - name: Validate base manifests | |
| run: | | |
| kubeconform -summary -strict -kubernetes-version 1.28.0 \ | |
| -ignore-filename-pattern 'kustomization.yaml' \ | |
| -ignore-filename-pattern 'traefik-ingressroute.yaml' \ | |
| k8s/base/*.yaml | |
| - name: Validate kustomized overlay output | |
| # Same shape as "Validate the storage root" below: rendered to a file | |
| # and checked non-empty, because this shell has no pipefail and | |
| # kubeconform reports success on empty input. | |
| run: | | |
| kubectl kustomize k8s/overlays/prod/ > /tmp/prod-lint.yaml | |
| if [ ! -s /tmp/prod-lint.yaml ]; then | |
| echo "::error::k8s/overlays/prod/ rendered nothing" | |
| exit 1 | |
| fi | |
| kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute \ | |
| < /tmp/prod-lint.yaml | |
| - name: Validate the CI overlay | |
| # k8s/overlays/ci/ is what the two kind jobs actually apply, so it is | |
| # the overlay whose breakage takes them down - and it is the one | |
| # nothing else renders: every static assertion reads prod on purpose, | |
| # because production sizing is the thing under test. | |
| # | |
| # kubeconform is only half of it. A `patches:` entry whose target | |
| # matches nothing is NOT an error in kustomize 5 - it renders happily | |
| # and silently applies nothing - so the day `rq-worker` is renamed in | |
| # the base, this overlay would keep validating while quietly handing | |
| # the kind jobs a full-sized 16Gi worker that no runner can schedule. | |
| # That failure is worth ten seconds here rather than an hour into a | |
| # kind job. Both properties are checked against the render: | |
| # 1. the worker really is smaller than prod's, i.e. the patch landed; | |
| # 2. requests still equal limits, so the shrink cannot introduce a | |
| # Burstable worker that makes assert_worker_qos_guaranteed report | |
| # a QoS regression existing nowhere but in CI's own sizing. | |
| run: | | |
| kubectl kustomize k8s/overlays/ci/ > /tmp/ci-rendered.yaml | |
| kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute \ | |
| < /tmp/ci-rendered.yaml | |
| kubectl kustomize k8s/overlays/prod/ > /tmp/prod-rendered.yaml | |
| WORKER_RESOURCES='select(.kind == "Deployment" and (.metadata.name | test("rq-worker$"))) | |
| | .spec.template.spec.containers[] | select(.name == "rq-worker") | .resources' | |
| CI_RES=$(yq "$WORKER_RESOURCES" /tmp/ci-rendered.yaml) | |
| PROD_RES=$(yq "$WORKER_RESOURCES" /tmp/prod-rendered.yaml) | |
| if [ -z "$CI_RES" ] || [ "$CI_RES" = "null" ]; then | |
| echo "::error::k8s/overlays/ci/ renders no rq-worker container resources - the patch target has drifted" | |
| exit 1 | |
| fi | |
| if [ "$CI_RES" = "$PROD_RES" ]; then | |
| echo "::error::k8s/overlays/ci/ renders the same worker size as prod, so its resource patch matched nothing; the kind jobs would deploy a worker no runner can schedule" | |
| echo "$CI_RES" | |
| exit 1 | |
| fi | |
| if [ "$(yq "$WORKER_RESOURCES | .requests" /tmp/ci-rendered.yaml)" != \ | |
| "$(yq "$WORKER_RESOURCES | .limits" /tmp/ci-rendered.yaml)" ]; then | |
| echo "::error::the CI worker has requests != limits, so it is Burstable; assert_worker_qos_guaranteed would fail on a regression that exists only in CI's sizing" | |
| echo "$CI_RES" | |
| exit 1 | |
| fi | |
| echo "CI worker sized down to:" | |
| echo "$CI_RES" | |
| - name: Validate the storage root | |
| # Rendered, not file-by-file: k8s/storage/ carries a Helm values file, | |
| # which is not a Kubernetes manifest and would fail kubeconform on | |
| # sight, and rendering is the only way the chart's own output gets | |
| # validated at all. --enable-helm is inert on a root with no helmCharts | |
| # field, so this keeps working if the chart is ever vendored. | |
| # | |
| # Rendered to a file rather than piped. `bash -e {0}` does not set | |
| # pipefail, so a failing kustomize on the left of a pipe was invisible: | |
| # this step printed the Helm 4 error, then | |
| # Summary: 0 resource found parsing stdin - Valid: 0 | |
| # and the job concluded success. The `[ -s ]` test is the load-bearing | |
| # half of the fix, not the redirect - kubeconform exits 0 on empty | |
| # input, so a render that SUCCEEDS and emits zero documents would still | |
| # pass a pipefail-corrected pipe. | |
| run: | | |
| kubectl kustomize --enable-helm k8s/storage/ > /tmp/storage-rendered.yaml | |
| if [ ! -s /tmp/storage-rendered.yaml ]; then | |
| echo "::error::k8s/storage/ rendered nothing; the chart was never validated" | |
| exit 1 | |
| fi | |
| kubeconform -summary -strict -kubernetes-version 1.28.0 \ | |
| < /tmp/storage-rendered.yaml | |
| assert-invariants: | |
| # Static assertions over the manifests themselves. This job deliberately | |
| # gates nothing - it has no `needs:` and nothing needs it - so that a red | |
| # invariant does not also hide the build and kind suites, which are the | |
| # only place cluster behaviour is observable. | |
| # | |
| # `assert_no_node_pinning_anywhere` is green as of the commit that deleted | |
| # k8s/components/memory-tier-*/nodeselector.yaml, and from here on it is a | |
| # ratchet: it exists to stop the pinning being reintroduced by a fork | |
| # rebase, which is exactly how it would come back. Its staying green is | |
| # not automatic - the NetworkPolicy step below renders the storage root, | |
| # and that render makes kustomize vendor the Ganesha chart, whose own | |
| # `nodeSelector` keys would otherwise be found by the pinning scan for | |
| # ever. k8s/storage/charts/ is .gitignored and the scan asks git for its | |
| # file list with `--exclude-standard`, so the two steps do not collide. | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install kubectl | |
| # Pinned to the version helm/kind-action installs in the kind jobs, so | |
| # every job in this file renders through the same kustomize (v5.7.1). | |
| # Unpinned, setup-kubectl@v3 resolves `latest` from the frozen legacy | |
| # GCS bucket - kubectl v1.31.0, kustomize 5.4.2 - and the static gates | |
| # then validate output the kind clusters never see. | |
| uses: azure/setup-kubectl@v3 | |
| with: | |
| version: v1.35.0 | |
| - name: Set up Helm | |
| # kubectl kustomize --enable-helm shells out to `helm` to inflate the | |
| # Ganesha chart in k8s/storage/. | |
| # | |
| # PINNED TO THE 3.x LINE ON PURPOSE, and pinned identically at all four | |
| # `Set up Helm` steps in this file. The kustomize built into kubectl | |
| # <= 1.35 (v5.7.1) probes the binary with `helm version -c --short` - a | |
| # shorthand Helm 4 removed - and then refuses any major that is not 3. | |
| # An unpinned `latest` resolved to Helm v4.2.4 and took every job that | |
| # renders k8s/storage/ down with | |
| # error: unknown shorthand flag: 'c' in -c | |
| # Do NOT "fix" this by bumping kubectl past 1.36 instead: kustomize | |
| # v5.8.1 stopped applying the namespace transformer to Helm-inflated | |
| # objects, so the chart's ServiceAccount, Service and StatefulSet would | |
| # render with no metadata.namespace - contradicting the contract stated | |
| # at the top of k8s/storage/kustomization.yaml, and silently deploying | |
| # Ganesha outside its NetworkPolicy for anyone using the documented | |
| # `kustomize | kubectl apply -f -` pipe. | |
| uses: azure/setup-helm@v4 | |
| with: | |
| # renovate: datasource=github-releases depName=helm/helm | |
| version: v3.21.4 | |
| - name: Check the assertion library | |
| run: | | |
| bash -n .github/scripts/ci-assertions.sh | |
| if command -v shellcheck >/dev/null; then | |
| shellcheck -S error .github/scripts/ci-assertions.sh | |
| fi | |
| - name: Assert the storage NetworkPolicy matches the overlay | |
| # Runs before the pinning check because it is the step that renders the | |
| # storage root, and the pinning scan's exclusion of the vendored chart | |
| # is the thing most likely to rot; having the render happen first means | |
| # the scan always runs against the state a real job leaves behind. | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_netpol_string_matches_overlay | |
| - name: Assert device-based fsids are pinned off | |
| # The static half of invariant 2. It lives here as well as inside | |
| # assert_stable_identity_across_restart so that it runs on every push, | |
| # in seconds, rather than only after a full kind deploy - the failure | |
| # it catches surfaces weeks later as ESTALE, so cheap and early is | |
| # worth more than thorough and late. | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_fsids_pinned | |
| - name: Assert the storage identity values are pinned | |
| # The other three quarters of invariant 2. device-based-fsids above is | |
| # only one of the four values that make the NFS server come back | |
| # serving the same bytes under the same identity; a values edit | |
| # dropping Retain, the fixed backing claim or the single replica | |
| # rendered clean, deployed clean and passed every runtime assertion, | |
| # and cost deleted data rather than a red build. | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_storage_identity_values | |
| - name: Assert no node pinning anywhere | |
| # Invariant 1: Kubernetes decides placement, the config must not. | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_no_node_pinning_anywhere | |
| build-amd64: | |
| # amd64 path. Produces per-arch tags `<ref>-<variant>-amd64`; the | |
| # multi-arch manifest under `<ref>-<variant>` (and `latest`) is stitched | |
| # together in `create-manifest` once the sibling `build-arm64` succeeds. | |
| needs: lint-manifests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - variant: full | |
| dockerfile: Dockerfile | |
| - variant: simple | |
| dockerfile: Dockerfile_simple | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Compute lowercase image name (OCI refs must be lowercase) | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GHCR | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch,suffix=-${{ matrix.variant }}-amd64 | |
| type=ref,event=tag,suffix=-${{ matrix.variant }}-amd64 | |
| type=sha,prefix=,suffix=-${{ matrix.variant }}-amd64 | |
| type=raw,value=latest-amd64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Build and conditionally push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ${{ matrix.dockerfile }} | |
| platforms: linux/amd64 | |
| load: true | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| # provenance/attestations turn the pushed tag into a manifest list, | |
| # which the create-manifest job's `docker manifest create` then | |
| # refuses ("is a manifest list"). Keep the push as a single-platform | |
| # image manifest — same as the build-arm64 job. | |
| provenance: false | |
| cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-amd64 | |
| cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-amd64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} | |
| build-args: | | |
| GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| - name: Retag for kind (image name the kustomize overlay points at) | |
| run: | | |
| # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, | |
| # `newTag: main-full`. The rendered manifests reference that exact | |
| # ref, so we need it loaded into kind under that name. Tag invariant | |
| # across branches/variants so the test always works. | |
| FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) | |
| docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full | |
| - name: Save image as tar | |
| run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar | |
| - name: Upload image artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: openms-streamlit-${{ matrix.variant }}-amd64-image | |
| path: /tmp/image.tar | |
| retention-days: 1 | |
| build-arm64: | |
| # arm64 path. Runs on a native ARM64 runner (no QEMU). Produces per-arch | |
| # tags `<ref>-<variant>-arm64`; gets merged into the multi-arch manifest | |
| # under `<ref>-<variant>` by the `create-manifest` job below. The build | |
| # uses a separate `Dockerfile.arm` / `Dockerfile_simple.arm` that swaps | |
| # the miniforge installer to aarch64 and (for the full variant) guards | |
| # the THIRDPARTY/Linux/aarch64 copy. The built image is also uploaded as | |
| # an artifact so the apptainer / nginx / traefik integration jobs can | |
| # exercise the ARM image on a native ARM runner (matrix arch=arm64). | |
| needs: lint-manifests | |
| runs-on: ubuntu-24.04-arm | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - variant: full | |
| dockerfile: Dockerfile.arm | |
| - variant: simple | |
| dockerfile: Dockerfile_simple.arm | |
| steps: | |
| - name: Free disk space | |
| # OpenMS source build needs ~25 GB of scratch space; the ARM runner | |
| # image is tighter than the AMD one out of the box. Mirrors what | |
| # FLASHApp's publish-docker-images.yml does at the top of its ARM job. | |
| run: | | |
| # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl | |
| # cache binaries there and fail if the directory is missing. | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true | |
| sudo apt-get clean | |
| df -h | |
| - uses: actions/checkout@v4 | |
| - name: Compute lowercase image name (OCI refs must be lowercase) | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GHCR | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch,suffix=-${{ matrix.variant }}-arm64 | |
| type=ref,event=tag,suffix=-${{ matrix.variant }}-arm64 | |
| type=sha,prefix=,suffix=-${{ matrix.variant }}-arm64 | |
| type=raw,value=latest-arm64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Build and conditionally push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ${{ matrix.dockerfile }} | |
| platforms: linux/arm64 | |
| load: true | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-arm64 | |
| cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-arm64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} | |
| provenance: false | |
| build-args: | | |
| GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| - name: Retag for kind (image name the kustomize overlay points at) | |
| run: | | |
| # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, | |
| # `newTag: main-full`. The rendered manifests reference that exact | |
| # ref, so we need it loaded into kind under that name. Tag invariant | |
| # across branches/variants so the test always works. | |
| FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) | |
| docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full | |
| - name: Save image as tar | |
| run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar | |
| - name: Upload image artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: openms-streamlit-${{ matrix.variant }}-arm64-image | |
| path: /tmp/image.tar | |
| retention-days: 1 | |
| create-manifest: | |
| # Stitch the per-arch tags into multi-arch manifest lists. The manifest | |
| # tags reuse the OLD scheme (`<ref>-<variant>`, `latest`) so existing | |
| # consumers (k8s overlays, docker-compose users, `docker pull` callers) | |
| # keep working transparently — docker now auto-selects the right arch | |
| # on pull. PRs don't push per-arch tags, so there's nothing to merge. | |
| needs: [build-amd64, build-arm64] | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| variant: [full, simple] | |
| steps: | |
| - name: Compute lowercase image name | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" | |
| - name: Log in to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Compute manifest tags | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| # NB: no -amd64/-arm64 suffix here. These are the multi-arch | |
| # manifest names; they must match the pre-arm64 tag scheme so | |
| # `:main-full`, `:v1.0.0-full`, `:latest` continue to resolve. | |
| tags: | | |
| type=ref,event=branch,suffix=-${{ matrix.variant }} | |
| type=ref,event=tag,suffix=-${{ matrix.variant }} | |
| type=sha,prefix=,suffix=-${{ matrix.variant }} | |
| type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Create and push multi-arch manifests | |
| # Iterate over manifest tags (newline-separated from metadata-action) | |
| # and merge the matching `-amd64` / `-arm64` per-arch tags into each. | |
| # `--amend` makes the step idempotent across workflow_dispatch reruns. | |
| # `docker manifest push` accepts only one ref per invocation, hence | |
| # the loop. | |
| run: | | |
| set -euo pipefail | |
| while IFS= read -r manifest_tag; do | |
| [ -z "$manifest_tag" ] && continue | |
| amd_tag="${manifest_tag}-amd64" | |
| arm_tag="${manifest_tag}-arm64" | |
| echo "Creating manifest ${manifest_tag} from:" | |
| echo " amd: ${amd_tag}" | |
| echo " arm: ${arm_tag}" | |
| docker manifest create "$manifest_tag" \ | |
| --amend "$amd_tag" \ | |
| --amend "$arm_tag" | |
| docker manifest push "$manifest_tag" | |
| done <<< "${{ steps.meta.outputs.tags }}" | |
| test-apptainer: | |
| # Apptainer/Singularity is the dominant container runtime on HPC clusters. | |
| # It mounts the root filesystem read-only and runs as the host user's UID | |
| # (not root inside the image). The entrypoint must tolerate both: this job | |
| # exercises that contract by running the built image under apptainer and | |
| # waiting for the streamlit /_stcore/health endpoint to come up. | |
| # | |
| # amd64 only: upstream apptainer does NOT publish arm64 .deb assets | |
| # (https://github.com/apptainer/apptainer/releases — every release lists | |
| # only `apptainer_<ver>_amd64.deb`), so eWaterCycle/setup-apptainer fails | |
| # on ubuntu-24.04-arm with "sudo exit code 100" when its | |
| # `apt-get install ./apptainer_*.deb` resolves a non-existent package. | |
| # Building apptainer from source on the arm runner would add ~15 min and | |
| # significant maintenance surface for limited value (HPC SIF consumers | |
| # remain amd64). Re-evaluate if upstream starts publishing arm64 builds. | |
| needs: build-amd64 | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| variant: [full, simple] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Free disk space | |
| # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind | |
| # node image plus loading the OCI tar into both docker and kind can | |
| # exhaust it. The arm runner is even tighter. Same incantation as | |
| # `build-arm64`'s "Free disk space" step. | |
| run: | | |
| # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl | |
| # cache binaries there and fail if the directory is missing. | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true | |
| sudo apt-get clean | |
| df -h | |
| - name: Download image artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: openms-streamlit-${{ matrix.variant }}-amd64-image | |
| path: /tmp | |
| - name: Install apptainer | |
| uses: eWaterCycle/setup-apptainer@v2 | |
| with: | |
| apptainer-version: 1.3.4 | |
| - name: Build SIF from docker-archive | |
| run: | | |
| sudo apptainer build /tmp/openms.sif docker-archive:///tmp/image.tar | |
| sudo chmod a+r /tmp/openms.sif | |
| - name: Prepare host bind dirs (mountpoint contract) | |
| run: | | |
| # Host paths we'll bind into the SIF. Asserting writability through | |
| # singularity's bind machinery requires that the destination paths | |
| # exist as real directories in the squashfs (otherwise singularity | |
| # silently degrades the bind to read-only via underlay). | |
| mkdir -p /tmp/host-workspaces /tmp/host-mounted-data | |
| echo "from-host-pretest" > /tmp/host-mounted-data/sentinel.txt | |
| - name: Start apptainer instance (read-only root, host UID, with binds) | |
| run: | | |
| # Default apptainer semantics: read-only root, no --writable-tmpfs. | |
| # This matches how users on HPC clusters run the SIF. | |
| # Use `instance run` (apptainer 1.1+), not `instance start`: the SIF | |
| # was built from docker-archive, which populates %runscript with the | |
| # Docker ENTRYPOINT but leaves %startscript as the default no-op | |
| # `exec "$@"`. `instance start` would launch an empty instance and | |
| # streamlit would never bind 8501. | |
| apptainer instance run \ | |
| --bind /tmp/host-workspaces:/workspaces-streamlit-template:rw \ | |
| --bind /tmp/host-mounted-data:/mounted-data:ro \ | |
| /tmp/openms.sif openms-test | |
| apptainer instance list | |
| # Record where this run's logs will land so subsequent steps can tail | |
| # them deterministically (path depends on hostname/user). | |
| LOG_DIR=$(find "$HOME/.apptainer/instances/logs" -type d -name "$(whoami)" 2>/dev/null | head -n 1) | |
| echo "APPTAINER_LOG_DIR=${LOG_DIR}" >> "$GITHUB_ENV" | |
| ls -la "$LOG_DIR" || true | |
| - name: Wait for streamlit /_stcore/health | |
| run: | | |
| # Tail the entrypoint's stdout/stderr alongside the health probe so | |
| # any startup failure surfaces directly in the CI log (the dedicated | |
| # "Dump entrypoint logs on failure" step is post-mortem only and | |
| # easy to miss in the GH Actions UI). | |
| OUT="${APPTAINER_LOG_DIR}/openms-test.out" | |
| ERR="${APPTAINER_LOG_DIR}/openms-test.err" | |
| for i in $(seq 1 90); do | |
| if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8501/_stcore/health; then | |
| echo "Streamlit is ready after $i attempts" | |
| exit 0 | |
| fi | |
| if [ $((i % 5)) -eq 0 ]; then | |
| echo "--- attempt $i: instance log tail ---" | |
| tail -n 20 "$OUT" 2>/dev/null || echo "(no $OUT yet)" | |
| tail -n 10 "$ERR" 2>/dev/null || echo "(no $ERR yet)" | |
| apptainer instance list || true | |
| fi | |
| sleep 2 | |
| done | |
| echo "TIMED OUT waiting for streamlit health endpoint" | |
| echo "--- full entrypoint stdout ---" | |
| cat "$OUT" 2>/dev/null || echo "(missing)" | |
| echo "--- full entrypoint stderr ---" | |
| cat "$ERR" 2>/dev/null || echo "(missing)" | |
| exit 1 | |
| - name: Verify health endpoint returns 200 | |
| run: curl -fsS http://127.0.0.1:8501/_stcore/health | |
| - name: Verify Redis is reachable inside container (full variant) | |
| if: matrix.variant == 'full' | |
| run: | | |
| # In apptainer mode the entrypoint uses a unix socket (TCP 6379 on | |
| # localhost is the host's, since net namespace is shared). The | |
| # entrypoint writes the resolved URL to /tmp/openms-redis-url for | |
| # out-of-band discovery, since `apptainer exec` spawns a fresh | |
| # shell that doesn't inherit the daemon's exported env. | |
| URL=$(apptainer exec instance://openms-test cat /tmp/openms-redis-url 2>/dev/null || true) | |
| case "$URL" in | |
| unix://*) | |
| SOCK="${URL#unix://}" | |
| echo "Redis URL is unix socket: $SOCK" | |
| apptainer exec instance://openms-test redis-cli -s "$SOCK" ping | grep -i pong | |
| ;; | |
| *) | |
| echo "Redis URL is TCP (or unset): ${URL:-default}" | |
| apptainer exec instance://openms-test redis-cli ping | grep -i pong | |
| ;; | |
| esac | |
| - name: Verify bind mount is writable (workspaces) and readable (data) | |
| run: | | |
| # The whole point of pre-creating /workspaces-streamlit-template | |
| # and /mounted-data in the image: singularity now has a real | |
| # attach point and `:rw` actually sticks. Without the mkdir, | |
| # `apptainer exec ... touch` here would fail with EROFS. | |
| apptainer exec instance://openms-test sh -c \ | |
| 'echo from-container > /workspaces-streamlit-template/probe.txt' | |
| test -f /tmp/host-workspaces/probe.txt | |
| grep -q from-container /tmp/host-workspaces/probe.txt | |
| # Read-only data mount should also be visible inside the container. | |
| apptainer exec instance://openms-test grep -q from-host-pretest /mounted-data/sentinel.txt | |
| # The mounted-drive browser uses os.path.ismount() to gate | |
| # rendering (existence is no longer enough now that the image | |
| # pre-creates the dir). Assert the kernel reports both paths as | |
| # real mount points so the detection function returns truthy. | |
| apptainer exec instance://openms-test python3 -c " | |
| import os, sys | |
| for p in ('/mounted-data', '/workspaces-streamlit-template'): | |
| assert os.path.ismount(p), f'{p} not reported as mount point' | |
| print(f'ismount({p}) = True') | |
| " | |
| - name: Dump entrypoint logs on failure | |
| if: failure() | |
| run: | | |
| echo "--- apptainer instance list ---" | |
| apptainer instance list || true | |
| echo "--- apptainer instance logs ---" | |
| find "$HOME/.apptainer" \( -name '*.out' -o -name '*.err' \) 2>/dev/null \ | |
| | while read -r f; do echo "=== $f ==="; cat "$f"; done || true | |
| - name: Stop apptainer instance | |
| if: always() | |
| run: apptainer instance stop openms-test || true | |
| - name: Upload validated SIF artifact (push events only) | |
| if: success() && github.event_name != 'pull_request' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: openms-streamlit-${{ matrix.variant }}-sif | |
| path: /tmp/openms.sif | |
| retention-days: 1 | |
| if-no-files-found: error | |
| publish-apptainer: | |
| # Publish the validated SIF (already health-checked above) to GHCR as an | |
| # OCI artifact via ORAS, in a sibling package: ghcr.io/<owner>/<repo>/sif. | |
| # Keeping it separate from the docker image package keeps tag lists clean | |
| # and lets HPC users `apptainer pull oras://...` without the 5-15 min | |
| # on-the-fly OCI->SIF conversion the docker:// path requires. | |
| needs: test-apptainer | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| variant: [full, simple] | |
| steps: | |
| - name: Download validated SIF artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: openms-streamlit-${{ matrix.variant }}-sif | |
| path: /tmp | |
| - name: Install apptainer | |
| uses: eWaterCycle/setup-apptainer@v2 | |
| with: | |
| apptainer-version: 1.3.4 | |
| - name: Compute SIF tags | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/sif | |
| tags: | | |
| type=ref,event=branch,suffix=-${{ matrix.variant }} | |
| type=ref,event=tag,suffix=-${{ matrix.variant }} | |
| type=sha,prefix=,suffix=-${{ matrix.variant }} | |
| type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| - name: Log in to GHCR for ORAS push | |
| env: | |
| GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # apptainer reads its auth from ~/.apptainer/remote.yaml, NOT from | |
| # ~/.docker/config.json — so docker/login-action won't work here. | |
| # Login and push must both run as the runner user (no sudo) so they | |
| # share the same $HOME and therefore the same auth file. | |
| echo "$GHCR_TOKEN" | apptainer registry login \ | |
| --username "${{ github.actor }}" \ | |
| --password-stdin \ | |
| oras://ghcr.io | |
| - name: Push SIF to each computed tag | |
| run: | | |
| # `apptainer push` accepts ONE destination per invocation; iterate | |
| # over the newline-separated tag list from docker/metadata-action. | |
| # tr lowercase is belt-and-braces — metadata-action already | |
| # lowercases, but GHCR is strict about case in OCI refs. | |
| set -euo pipefail | |
| while IFS= read -r tag; do | |
| [ -z "$tag" ] && continue | |
| tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" | |
| echo "Pushing SIF to oras://${tag_lc}" | |
| apptainer push /tmp/openms.sif "oras://${tag_lc}" | |
| done <<< "${{ steps.meta.outputs.tags }}" | |
| test-nginx: | |
| needs: [build-amd64, build-arm64] | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - variant: full | |
| arch: amd64 | |
| runner: ubuntu-latest | |
| - variant: full | |
| arch: arm64 | |
| runner: ubuntu-24.04-arm | |
| - variant: simple | |
| arch: amd64 | |
| runner: ubuntu-latest | |
| - variant: simple | |
| arch: arm64 | |
| runner: ubuntu-24.04-arm | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Free disk space | |
| # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind | |
| # node image plus loading the OCI tar into both docker and kind can | |
| # exhaust it. The arm runner is even tighter. Same incantation as | |
| # `build-arm64`'s "Free disk space" step. | |
| run: | | |
| # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl | |
| # cache binaries there and fail if the directory is missing. | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true | |
| sudo apt-get clean | |
| df -h | |
| - name: Download image artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image | |
| path: /tmp | |
| - name: Create kind cluster | |
| uses: helm/kind-action@v1 | |
| with: | |
| # Pinned to what the action picks today. kubectl_version must stay | |
| # equal to the `Install kubectl` pin above, or the static jobs and | |
| # the kind jobs render k8s/storage/ through different kustomize | |
| # versions and disagree about namespaces on Helm-inflated objects. | |
| # node_image is deliberately NOT pinned: a digest that goes stale | |
| # fails cluster creation outright, with no fallback. | |
| version: v0.31.0 | |
| kubectl_version: v1.35.0 | |
| cluster_name: test-cluster | |
| config: .github/kind-config.yaml | |
| - name: Load image into kind cluster | |
| # Use `kind load image-archive` (not docker-image) so we never store | |
| # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete | |
| # the tar afterwards to free the same again on /tmp — the image is | |
| # now in both kind nodes' containerd, which is enough. | |
| run: | | |
| kind load image-archive /tmp/image.tar --name test-cluster | |
| rm -f /tmp/image.tar | |
| # Diagnostic only. The image has just been written into both kind | |
| # nodes' containerd, and nfs-provisioner refuses claims larger than | |
| # the free space under /export - so when a claim fails to bind, this | |
| # line is the difference between "out of disk" and a lost afternoon. | |
| df -h / || true | |
| - name: Install nginx ingress controller | |
| run: | | |
| kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml | |
| kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s | |
| - name: Set up Helm | |
| # kubectl kustomize --enable-helm shells out to `helm` to inflate the | |
| # Ganesha chart in k8s/storage/. | |
| # | |
| # PINNED TO THE 3.x LINE ON PURPOSE, and pinned identically at all four | |
| # `Set up Helm` steps in this file. The kustomize built into kubectl | |
| # <= 1.35 (v5.7.1) probes the binary with `helm version -c --short` - a | |
| # shorthand Helm 4 removed - and then refuses any major that is not 3. | |
| # An unpinned `latest` resolved to Helm v4.2.4 and took every job that | |
| # renders k8s/storage/ down with | |
| # error: unknown shorthand flag: 'c' in -c | |
| # Do NOT "fix" this by bumping kubectl past 1.36 instead: kustomize | |
| # v5.8.1 stopped applying the namespace transformer to Helm-inflated | |
| # objects, so the chart's ServiceAccount, Service and StatefulSet would | |
| # render with no metadata.namespace - contradicting the contract stated | |
| # at the top of k8s/storage/kustomization.yaml, and silently deploying | |
| # Ganesha outside its NetworkPolicy for anyone using the documented | |
| # `kustomize | kubectl apply -f -` pipe. | |
| uses: azure/setup-helm@v4 | |
| with: | |
| # renovate: datasource=github-releases depName=helm/helm | |
| version: v3.21.4 | |
| - name: Report the NFS client helper on the kind nodes | |
| # Diagnostic, not a gate - the gate is whether the claim binds and the | |
| # pods come up. An in-tree `nfs:` PV is mounted by the kubelet on the | |
| # node, which needs /sbin/mount.nfs there; without this line a missing | |
| # helper surfaces several steps later as "wrong fs type, bad option, | |
| # bad superblock" in a pod event. Same probe as A16-RUNBOOK.md section 0 | |
| # runs against the real nodes. | |
| run: | | |
| for n in $(kind get nodes --name test-cluster); do | |
| echo "--- $n" | |
| docker exec "$n" sh -c 'ls -l /sbin/mount.nfs* /usr/sbin/mount.nfs* 2>&1; echo ---; grep nfs /proc/filesystems' || true | |
| done | |
| - name: Deploy the storage root (NFS-Ganesha) | |
| # Before the overlay, because workspaces-pvc is ReadWriteMany on the | |
| # StorageClass this root publishes: with no provisioner running the | |
| # claim never binds and every app pod stays Pending. Ganesha is just a | |
| # pod on a PVC and kind's `standard` class is RWO local-path, so the | |
| # whole NFS re-export genuinely runs here. | |
| run: | | |
| # Nothing vendors the chart - .gitignore excludes k8s/storage/charts/ | |
| # on purpose - so every job re-fetches index.yaml and the tarball from | |
| # GitHub Pages. A transient 5xx there presents as exactly the same | |
| # "kubectl kustomize --enable-helm failed" this pipeline has already | |
| # been taken down by once, so retry the render rather than the apply | |
| # alone. | |
| rendered=0 | |
| for i in 1 2 3; do | |
| if kubectl kustomize --enable-helm k8s/storage/ > /tmp/storage-rendered.yaml; then | |
| rendered=1 | |
| break | |
| fi | |
| echo "Render attempt $i failed, retrying in ${i}0s..." | |
| sleep "${i}0" | |
| done | |
| if [ "$rendered" -ne 1 ] || [ ! -s /tmp/storage-rendered.yaml ]; then | |
| echo "::error::could not render k8s/storage/ (needs Helm 3.x on PATH)" | |
| exit 1 | |
| fi | |
| # Ganesha's own backing claim is now the only Cinder-classed object in | |
| # the tree. kind has no cinder-csi, so rewrite it to `standard`, and | |
| # fail loudly if the string is gone rather than letting the rewrite | |
| # no-op and quietly take the storage path out of CI with it. | |
| if ! grep -q 'storageClassName: cinder-csi' /tmp/storage-rendered.yaml; then | |
| echo "::error::k8s/storage/ no longer renders 'storageClassName: cinder-csi'; the rewrite below would silently no-op" | |
| exit 1 | |
| fi | |
| # 500Gi is right for de.NBI and absurd on a runner. local-path never | |
| # checks capacity so it binds anyway, but nfs-provisioner statfs()es | |
| # /export and refuses any workspace claim larger than the free space | |
| # it reports - and a 500Gi Bound claim in the failure dump sends the | |
| # next reader looking in the wrong place entirely. | |
| if ! grep -q 'storage: 500Gi' /tmp/storage-rendered.yaml; then | |
| echo "::error::k8s/storage/ no longer renders 'storage: 500Gi'; the size rewrite below would silently no-op" | |
| exit 1 | |
| fi | |
| sed -e 's|storageClassName: cinder-csi|storageClassName: standard|g' \ | |
| -e 's|storage: 500Gi|storage: 8Gi|' \ | |
| /tmp/storage-rendered.yaml > /tmp/storage.yaml | |
| # The node CIDR in networkpolicy.yaml ships as RFC 5737 TEST-NET-1 and | |
| # is documented there as a placeholder to be set before the first | |
| # deploy. It is not cosmetic here: kindnetd runs kube-network-policies | |
| # unconditionally, the namespace default-deny therefore applies, and | |
| # the in-tree `nfs:` PVs this chart emits are mounted by the KUBELET | |
| # from the node's own address in the host netns - matching no | |
| # podSelector and no ipBlock. Left alone, every mount on a node not | |
| # running Ganesha hangs, which the worker's DoNotSchedule spread | |
| # guarantees will happen to exactly one replica. | |
| if ! grep -q 'cidr: 192.0.2.0/24' /tmp/storage.yaml; then | |
| echo "::error::k8s/storage/networkpolicy.yaml no longer carries the 192.0.2.0/24 placeholder; this rewrite would silently no-op and every cross-node mount would hang" | |
| exit 1 | |
| fi | |
| # Derived, not hardcoded: kind pins only its IPv6 subnet, so the IPv4 | |
| # one comes from Docker's default address pool and is not 172.18/16 | |
| # by contract. | |
| NODE_CIDR=$(docker network inspect kind \ | |
| -f '{{range .IPAM.Config}}{{.Subnet}} {{end}}' \ | |
| | tr ' ' '\n' | grep -v ':' | grep -E '^[0-9.]+/[0-9]+$' | head -n1) | |
| if [ -z "$NODE_CIDR" ]; then | |
| echo "::error::could not read an IPv4 subnet from the kind docker network" | |
| docker network inspect kind -f '{{json .IPAM.Config}}' | |
| exit 1 | |
| fi | |
| echo "admitting kind nodes on 2049 from $NODE_CIDR" | |
| sed -i "s|cidr: 192.0.2.0/24|cidr: ${NODE_CIDR}|" /tmp/storage.yaml | |
| # kubeconform validated /tmp/storage-rendered.yaml, not the file that | |
| # is actually applied. Re-check the one value the seds above compute | |
| # rather than copy, so a malformed CIDR fails here and names itself | |
| # instead of surfacing as an API-server rejection mid-retry-loop. | |
| if ! grep -Eq '^[[:space:]]*cidr: [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$' /tmp/storage.yaml; then | |
| echo "::error::the rewritten NetworkPolicy CIDR is not a bare IPv4 block" | |
| grep -n 'cidr:' /tmp/storage.yaml | |
| exit 1 | |
| fi | |
| applied=0 | |
| for i in 1 2 3 4 5; do | |
| if kubectl apply -f /tmp/storage.yaml; then | |
| applied=1 | |
| echo "Storage apply succeeded on attempt $i" | |
| break | |
| fi | |
| echo "Attempt $i failed, retrying in ${i}0s..." | |
| sleep "${i}0" | |
| done | |
| # `if ... then break; fi` never trips `set -e`, so without this the | |
| # loop fell through on total failure and the step died on the bare | |
| # NotFound from the substitution below - taking the authored error | |
| # message and its diagnostic dump with it, in precisely the case the | |
| # retry loop exists to report. | |
| if [ "$applied" -ne 1 ]; then | |
| echo "::error::could not apply k8s/storage/ after 5 attempts" | |
| kubectl get all -n template-app-storage || true | |
| exit 1 | |
| fi | |
| # `kubectl wait --all` reports success against zero objects, so the | |
| # server has to be found before it is waited on. `|| true` because an | |
| # empty result must reach the explicit check below rather than kill | |
| # the step through `bash -e`. | |
| WORKLOADS=$(kubectl get deploy,statefulset -n template-app-storage -o name 2>/dev/null || true) | |
| if [ -z "$WORKLOADS" ]; then | |
| echo "::error::k8s/storage/ created no Deployment or StatefulSet in template-app-storage" | |
| kubectl get all -n template-app-storage | |
| exit 1 | |
| fi | |
| for w in $WORKLOADS; do | |
| kubectl rollout status -n template-app-storage "$w" --timeout=300s | |
| done | |
| kubectl get pods,pvc -n template-app-storage -o wide | |
| kubectl get storageclass | |
| # Diagnostic, never a gate: this is the filesystem nfs-provisioner | |
| # measures when it decides whether a workspace claim fits. | |
| kubectl exec -n template-app-storage \ | |
| "$(kubectl get pod -n template-app-storage -l app=nfs-server -o name | head -n1)" \ | |
| -- df -h /export || true | |
| - name: Deploy with Kustomize | |
| run: | | |
| # k8s/overlays/ci/, not prod: prod's rq-worker is 16Gi / 4 cpu with | |
| # requests == limits, which is right on the de.NBI nodes and | |
| # impossible here. This runner has 4 vCPU and ~15.6Gi in total, every | |
| # kind node is a container on that one host advertising the whole of | |
| # it, and kube-system already holds ~1 cpu in requests - so a single | |
| # replica does not fit, let alone two, and both would sit Pending | |
| # until "Verify all deployments are available" failed at 180s and | |
| # took every assertion below it with it. The CI overlay is prod with | |
| # exactly one thing changed: the worker's size. Same replica count, | |
| # same spread constraint, same labels, same images, so everything | |
| # this suite observes is still the production arrangement. The lint | |
| # job checks that the shrink actually applied and stayed Guaranteed. | |
| # | |
| # Filter out Traefik IngressRoute (kind cluster uses nginx) and force imagePullPolicy=Never | |
| # No cinder-csi rewrite here any more. The only Cinder-classed | |
| # object left in the tree is Ganesha's own backing claim in | |
| # k8s/storage/, and the storage step above rewrites that one - loudly | |
| # if the string ever disappears. A sed kept here would match nothing | |
| # and quietly imply the storage path was still being exercised. | |
| kubectl kustomize k8s/overlays/ci/ | \ | |
| yq 'select(.kind != "IngressRoute")' | \ | |
| sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' > /tmp/manifests.yaml | |
| # Size the workspaces claim for the runner's disk. nfs-provisioner | |
| # statfs()es /export on every provision and refuses outright any | |
| # claim larger than the free space it finds there - the check is | |
| # unconditional, not gated on quotas. In kind the export is a | |
| # local-path volume on the GitHub runner's disk, tens of GiB, so the | |
| # production figure could never bind and every pod would sit Pending | |
| # behind it. Fail loudly if the request has gone, rather than letting | |
| # the rewrite no-op and take the storage path out of CI with it. | |
| if ! grep -qE 'storage: [0-9]+Gi' /tmp/manifests.yaml; then | |
| echo "::error::the rendered overlay has no 'storage: <N>Gi' request; the rewrite below would silently no-op" | |
| exit 1 | |
| fi | |
| # 1Gi, not 4Gi: nfs-provisioner statfs()es /export and refuses any | |
| # claim larger than the free space it reports, and /export here is a | |
| # local-path directory on a runner disk that has just absorbed the | |
| # app image into both kind nodes' containerd. No assertion in this | |
| # suite writes anywhere near a gigabyte - the largest is ~60 short | |
| # log lines - so the headroom buys nothing and costs a 300s Bound | |
| # timeout whose message is about bytes. | |
| sed -i -E 's|storage: [0-9]+Gi|storage: 1Gi|g' /tmp/manifests.yaml | |
| for i in 1 2 3 4 5; do | |
| if kubectl apply -f /tmp/manifests.yaml; then | |
| echo "Deploy succeeded on attempt $i" | |
| break | |
| fi | |
| echo "Attempt $i failed, retrying in ${i}0s..." | |
| sleep "${i}0" | |
| done | |
| - name: Discover overlay identity | |
| # Still read off prod, even though ci/ is what was applied: the CI | |
| # overlay has no commonLabels of its own, it inherits prod's whole | |
| # identity and only repatches the worker's resources. Reading the | |
| # source of that identity keeps this working if the CI overlay ever | |
| # grows a second patch. | |
| run: | | |
| SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) | |
| echo "SLUG=$SLUG" >> "$GITHUB_ENV" | |
| - name: Verify the workspace StorageClass contract | |
| # k8s/base/workspace-pvc.yaml names a class k8s/storage/ has to publish, | |
| # and the mount options ride on that class because a PVC has nowhere to | |
| # carry them. Checking all of it here turns drift into one failed step | |
| # with the actual values printed, instead of a Pending pod at 3am. | |
| run: | | |
| # Read the claim off the Deployment that mounts it, so namePrefix and | |
| # any second PVC in the namespace stay irrelevant. | |
| CLAIM=$(kubectl get deployment -n openms -l app=${SLUG},component=streamlit \ | |
| -o jsonpath='{.items[0].spec.template.spec.volumes[?(@.name=="workspaces")].persistentVolumeClaim.claimName}') | |
| if [ -z "${CLAIM}" ]; then | |
| echo "::error::no streamlit Deployment in openms mounts a 'workspaces' volume - there is no claim to check" | |
| kubectl get deployment -n openms -o wide | |
| exit 1 | |
| fi | |
| kubectl wait -n openms --for=jsonpath='{.status.phase}'=Bound \ | |
| "pvc/${CLAIM}" --timeout=300s | |
| MODES=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.status.accessModes[*]}') | |
| case " ${MODES} " in | |
| *" ReadWriteMany "*) ;; | |
| *) | |
| echo "::error::${CLAIM} bound as '${MODES}', not ReadWriteMany; every pod that mounts it is back to sharing one node" | |
| exit 1 | |
| ;; | |
| esac | |
| CLASS=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.spec.storageClassName}') | |
| echo "${CLAIM} is Bound ReadWriteMany on StorageClass '${CLASS}'" | |
| if ! kubectl get storageclass "${CLASS}" >/dev/null 2>&1; then | |
| echo "::error::the workspaces PVC names StorageClass '${CLASS}', which k8s/storage/ does not publish" | |
| kubectl get storageclass | |
| exit 1 | |
| fi | |
| OPTS=$(kubectl get storageclass "${CLASS}" -o jsonpath='{.mountOptions[*]}') | |
| echo "mount options: ${OPTS:-<none>}" | |
| case " ${OPTS} " in | |
| *" soft "*) | |
| echo "::error::'${CLASS}' mounts soft; a routine Ganesha restart would then truncate in-flight writes instead of blocking" | |
| exit 1 | |
| ;; | |
| *" hard "*) ;; | |
| *) | |
| echo "::error::'${CLASS}' does not pin 'hard'. It is also the kernel default, but the design states it explicitly so that adding 'soft' is a visible diff" | |
| exit 1 | |
| ;; | |
| esac | |
| echo "${OPTS}" | grep -qE '(^| )(nfs)?vers=4\.1( |$)' || { | |
| echo "::error::'${CLASS}' does not pin NFSv4.1, which is what supplies integrated locking on one port" | |
| exit 1 | |
| } | |
| echo "${OPTS}" | grep -qE '(^| )nconnect=4( |$)' || { | |
| echo "::error::'${CLASS}' does not set nconnect=4" | |
| exit 1 | |
| } | |
| if echo "${OPTS}" | grep -qE '(^| )(actimeo|acregmin|acregmax|acdirmin|acdirmax)='; then | |
| echo "::error::'${CLASS}' overrides the attribute cache timeouts; close-to-open consistency already covers this app, so actimeo stays at its default" | |
| exit 1 | |
| fi | |
| - name: Wait for Redis to be ready | |
| run: | | |
| kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s | |
| - name: Verify Redis Service is reachable | |
| run: | | |
| kubectl run redis-test -n openms --image=redis:7-alpine --rm -i --restart=Never -- redis-cli -h ${SLUG}-redis.openms.svc.cluster.local ping | |
| - name: Verify all deployments are available | |
| # No `|| true`: a Pending, CrashLooping or unschedulable Deployment has to | |
| # fail the job, otherwise the ingress curl below is the only real gate. | |
| # Diagnostics come from "Dump cluster state on failure". | |
| run: | | |
| kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s | |
| kubectl get pods -n openms -l app=${SLUG} -o wide | |
| kubectl get services -n openms -l app=${SLUG} | |
| - name: Curl both hostnames via nginx ingress | |
| run: | | |
| NGINX_POD=$(kubectl -n ingress-nginx get pod -l app.kubernetes.io/component=controller -o name | head -n 1) | |
| kubectl -n ingress-nginx port-forward "$NGINX_POD" 8080:80 & | |
| PF_PID=$! | |
| trap 'kill "$PF_PID" 2>/dev/null || true' EXIT | |
| for i in $(seq 1 30); do | |
| sleep 2 | |
| if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8080/_stcore/health -H "Host: streamlit.openms.example.de"; then | |
| break | |
| fi | |
| echo "port-forward / app not ready yet, retry $i" | |
| done | |
| for host in streamlit.openms.example.de streamlit.openms.example.org; do | |
| curl -fsS --resolve "$host:8080:127.0.0.1" "http://$host:8080/_stcore/health" | |
| echo "" | |
| echo "$host -> 200 OK" | |
| done | |
| # Everything below is `.github/scripts/ci-assertions.sh`. Each assertion | |
| # discovers the namespace, the app label, the workspace claim and the | |
| # helper image from the cluster, so SLUG above is the only input any of | |
| # them needs. | |
| # | |
| # The NetworkPolicy assertion is first, and then the two placement ones, | |
| # because all three are seconds long where the storage ones are up to 45 | |
| # minutes of deliberate timeouts. They read the state that "Verify all | |
| # deployments are available" has just established, and if the workers are | |
| # not where they should be then everything after this is noise measured | |
| # against a broken deployment. | |
| - name: Assert every node is admitted to the NFS share | |
| # First, because it is the failure that disguises itself as all the | |
| # others. The share is behind a namespace-wide default-deny, the | |
| # in-tree `nfs:` PVs are mounted by the KUBELET from the node address, | |
| # and the CIDR that admits them is rewritten by the storage deploy step | |
| # rather than shipped - so if that rewrite ever no-ops, every mount on | |
| # every node except Ganesha's own hangs, and the first thing anyone | |
| # sees is an unrelated assertion timing out forty minutes later. | |
| timeout-minutes: 2 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_netpol_admits_every_node | |
| - name: Assert the worker replicas are spread across nodes | |
| # Step 4's acceptance criterion for the workers. Every replica Running | |
| # - so an unschedulable one fails here rather than being left out of | |
| # the node count - on more than one node, inside the maxSkew the | |
| # Deployment itself declares, under a spread constraint whose | |
| # labelSelector really selects these workers. | |
| # | |
| # assert_two_pods_two_nodes below does NOT cover this: it starts its | |
| # own helper pods with an explicit nodeName, which proves the volume is | |
| # reachable from two nodes and says nothing about where the scheduler | |
| # put the rq-workers. Deleting the nodeSelector is necessary for this | |
| # to pass and is not sufficient - a spread constraint scoped to the | |
| # wrong labels, or dropped entirely, fails right here. | |
| timeout-minutes: 5 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_workers_spread_across_nodes | |
| - name: Assert the workers are Guaranteed QoS | |
| # requests == limits, read off `.status.qosClass` on the RUNNING pods. | |
| # A Burstable worker scores oom_score_adj ~969 and sits near the top of | |
| # the node's kill list holding hours of work; the regression that | |
| # produces it is a one-line edit to a memory-tier component and is | |
| # otherwise invisible until something OOMKills under load. | |
| timeout-minutes: 5 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_worker_qos_guaranteed | |
| - name: Assert cross-node write visibility | |
| # Two pods holding the same claim on two different nodes at the same | |
| # time, each reading what the other wrote. | |
| timeout-minutes: 10 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_cross_node_write_visible | |
| - name: Assert the POSIX contract holds on the shared volume | |
| # Absolute symlinks, atomic rename and flock - what src/ already | |
| # depends on and what NFS is allowed to take away. It first checks the | |
| # mount really is NFS, because every one of those passes against the | |
| # node's local filesystem too, and the flock half runs across two pods | |
| # on two nodes, because a lock that only excludes a second holder on | |
| # the same node is a lock the two workers do not have. | |
| timeout-minutes: 15 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_posix_contract | |
| - name: Assert a workflow survives an NFS server restart | |
| # The single most valuable assertion in the set: delete the Ganesha pod | |
| # while a workflow is running and assert the workflow COMPLETES, with | |
| # every line of its output present and in order. A Ganesha restart is | |
| # routine, and this is exactly the case a `soft` mount would have | |
| # corrupted - into a truncated featureXML rather than a failed job. | |
| # Slow by construction: the restart and the ~90s NFSv4.1 grace period | |
| # both pass underneath a running job, which is the point. | |
| timeout-minutes: 20 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_survives_nfs_restart | |
| - name: Assert stable identity across an NFS server restart | |
| # Invariant 2. Deletes the Ganesha pod under a live client, so it is | |
| # slow by construction: NFSv4.1 spends ~90s in its grace period. | |
| timeout-minutes: 15 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_stable_identity_across_restart | |
| - name: Assert two pods on two nodes share the workspace volume | |
| # The acceptance criterion for the whole exercise, and the last of these | |
| # to go green: it stayed red until the memory-tier nodeSelector patches | |
| # were deleted. Everything above it is about the export itself and | |
| # passes as soon as ReadWriteMany works, which is what kept "RWX proven | |
| # while still single-node" observable. | |
| # | |
| # It carried `continue-on-error: true` while it was red by design - a | |
| # step that always fails makes every genuine regression around it | |
| # indistinguishable from the expected red, because nobody looks at an | |
| # already-failing job. That commit has landed, so the criterion gates | |
| # like everything else here. | |
| timeout-minutes: 10 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_two_pods_two_nodes | |
| - name: Dump cluster state on failure | |
| if: failure() | |
| run: | | |
| echo "=== nodes ===" | |
| kubectl get nodes -o wide || true | |
| echo "=== pods (all namespaces) ===" | |
| kubectl get pods -A -o wide || true | |
| echo "=== storage tier ===" | |
| kubectl get all,pvc -n template-app-storage -o wide || true | |
| kubectl describe pods -n template-app-storage || true | |
| for p in $(kubectl get pods -n template-app-storage -o name 2>/dev/null); do | |
| kubectl logs -n template-app-storage "$p" --all-containers --prefix --tail=200 || true | |
| done | |
| echo "=== storage classes and volumes ===" | |
| kubectl get storageclass || true | |
| kubectl get pv,pvc -A || true | |
| # Unfiltered on purpose. SLUG is exported by "Discover overlay | |
| # identity", which runs AFTER the storage steps - so any failure | |
| # before it left ${SLUG} empty here, every `-l app=` matched nothing, | |
| # and this dump reported an empty namespace on a cluster that simply | |
| # had not been deployed to yet. In CI the openms namespace holds only | |
| # this app, so dropping the selector cannot lose anything. | |
| echo "=== app pods describe ===" | |
| kubectl describe pod -n openms || true | |
| echo "=== app pod logs ===" | |
| for p in $(kubectl get pods -n openms -o name 2>/dev/null); do | |
| kubectl logs -n openms "$p" --all-containers --prefix --tail=200 || true | |
| done | |
| echo "=== app pod previous logs (if crashed) ===" | |
| for p in $(kubectl get pods -n openms -o name 2>/dev/null); do | |
| kubectl logs -n openms "$p" --all-containers --prefix --tail=200 --previous || true | |
| done | |
| echo "=== ingress ===" | |
| kubectl get ingress -A -o wide || true | |
| kubectl describe ingress -n openms || true | |
| echo "=== services + endpoints ===" | |
| kubectl get svc,endpoints -n openms || true | |
| echo "=== ingress-nginx controller logs ===" | |
| kubectl logs -n ingress-nginx -l app.kubernetes.io/component=controller --tail=200 || true | |
| test-traefik: | |
| needs: [build-amd64, build-arm64] | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - variant: full | |
| arch: amd64 | |
| runner: ubuntu-latest | |
| - variant: full | |
| arch: arm64 | |
| runner: ubuntu-24.04-arm | |
| - variant: simple | |
| arch: amd64 | |
| runner: ubuntu-latest | |
| - variant: simple | |
| arch: arm64 | |
| runner: ubuntu-24.04-arm | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Free disk space | |
| # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind | |
| # node image plus loading the OCI tar into both docker and kind can | |
| # exhaust it. The arm runner is even tighter. Same incantation as | |
| # `build-arm64`'s "Free disk space" step. | |
| run: | | |
| # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl | |
| # cache binaries there and fail if the directory is missing. | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true | |
| sudo apt-get clean | |
| df -h | |
| - name: Download image artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image | |
| path: /tmp | |
| - name: Create kind cluster | |
| uses: helm/kind-action@v1 | |
| with: | |
| # Keep in lockstep with the nginx job and the `Install kubectl` pins. | |
| version: v0.31.0 | |
| kubectl_version: v1.35.0 | |
| cluster_name: traefik-test | |
| config: .github/kind-config.yaml | |
| - name: Load image into kind cluster | |
| # Use `kind load image-archive` (not docker-image) so we never store | |
| # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete | |
| # the tar afterwards to free the same again on /tmp — the image is | |
| # now in both kind nodes' containerd, which is enough. | |
| run: | | |
| kind load image-archive /tmp/image.tar --name traefik-test | |
| rm -f /tmp/image.tar | |
| # Diagnostic only. The image has just been written into both kind | |
| # nodes' containerd, and nfs-provisioner refuses claims larger than | |
| # the free space under /export - so when a claim fails to bind, this | |
| # line is the difference between "out of disk" and a lost afternoon. | |
| df -h / || true | |
| - name: Set up Helm | |
| # Same 3.x pin as the other three `Set up Helm` steps in this file, and | |
| # for the same reason - see the long note on the first of them. This one | |
| # also drives `helm install traefik` below; chart 41.x declares a | |
| # `Helm v3.9.0+` prerequisite, which v3.21.4 clears comfortably. | |
| uses: azure/setup-helm@v4 | |
| with: | |
| # renovate: datasource=github-releases depName=helm/helm | |
| version: v3.21.4 | |
| - name: Install Traefik via Helm | |
| run: | | |
| helm repo add traefik https://traefik.github.io/charts | |
| helm repo update | |
| # `service.type` is NOT a key in this chart and never has been - the | |
| # Service type lives at `service.spec.type`, defaulting to | |
| # LoadBalancer. The old --set was a silent no-op that nothing caught, | |
| # because a port-forward works against a LoadBalancer Service too. | |
| # renovate: datasource=helm registryUrl=https://traefik.github.io/charts depName=traefik | |
| helm install traefik traefik/traefik \ | |
| --version 41.3.0 \ | |
| --namespace traefik --create-namespace \ | |
| --set service.spec.type=ClusterIP | |
| kubectl -n traefik wait --for=condition=available deployment/traefik --timeout=120s | |
| # Assert the --set actually landed, so the next time this chart moves | |
| # its values path it fails here by name instead of silently reverting | |
| # to the default. | |
| SVC_TYPE=$(kubectl -n traefik get svc traefik -o jsonpath='{.spec.type}') | |
| if [ "$SVC_TYPE" != "ClusterIP" ]; then | |
| echo "::error::traefik Service is ${SVC_TYPE}, not ClusterIP; the chart's values path for the Service type has moved again" | |
| exit 1 | |
| fi | |
| - name: Report the NFS client helper on the kind nodes | |
| # Diagnostic, not a gate - the gate is whether the claim binds and the | |
| # pods come up. An in-tree `nfs:` PV is mounted by the kubelet on the | |
| # node, which needs /sbin/mount.nfs there; without this line a missing | |
| # helper surfaces several steps later as "wrong fs type, bad option, | |
| # bad superblock" in a pod event. Same probe as A16-RUNBOOK.md section 0 | |
| # runs against the real nodes. | |
| run: | | |
| for n in $(kind get nodes --name traefik-test); do | |
| echo "--- $n" | |
| docker exec "$n" sh -c 'ls -l /sbin/mount.nfs* /usr/sbin/mount.nfs* 2>&1; echo ---; grep nfs /proc/filesystems' || true | |
| done | |
| - name: Deploy the storage root (NFS-Ganesha) | |
| # Before the overlay, because workspaces-pvc is ReadWriteMany on the | |
| # StorageClass this root publishes: with no provisioner running the | |
| # claim never binds and every app pod stays Pending. Ganesha is just a | |
| # pod on a PVC and kind's `standard` class is RWO local-path, so the | |
| # whole NFS re-export genuinely runs here. | |
| run: | | |
| # Nothing vendors the chart - .gitignore excludes k8s/storage/charts/ | |
| # on purpose - so every job re-fetches index.yaml and the tarball from | |
| # GitHub Pages. A transient 5xx there presents as exactly the same | |
| # "kubectl kustomize --enable-helm failed" this pipeline has already | |
| # been taken down by once, so retry the render rather than the apply | |
| # alone. | |
| rendered=0 | |
| for i in 1 2 3; do | |
| if kubectl kustomize --enable-helm k8s/storage/ > /tmp/storage-rendered.yaml; then | |
| rendered=1 | |
| break | |
| fi | |
| echo "Render attempt $i failed, retrying in ${i}0s..." | |
| sleep "${i}0" | |
| done | |
| if [ "$rendered" -ne 1 ] || [ ! -s /tmp/storage-rendered.yaml ]; then | |
| echo "::error::could not render k8s/storage/ (needs Helm 3.x on PATH)" | |
| exit 1 | |
| fi | |
| # Ganesha's own backing claim is now the only Cinder-classed object in | |
| # the tree. kind has no cinder-csi, so rewrite it to `standard`, and | |
| # fail loudly if the string is gone rather than letting the rewrite | |
| # no-op and quietly take the storage path out of CI with it. | |
| if ! grep -q 'storageClassName: cinder-csi' /tmp/storage-rendered.yaml; then | |
| echo "::error::k8s/storage/ no longer renders 'storageClassName: cinder-csi'; the rewrite below would silently no-op" | |
| exit 1 | |
| fi | |
| # 500Gi is right for de.NBI and absurd on a runner. local-path never | |
| # checks capacity so it binds anyway, but nfs-provisioner statfs()es | |
| # /export and refuses any workspace claim larger than the free space | |
| # it reports - and a 500Gi Bound claim in the failure dump sends the | |
| # next reader looking in the wrong place entirely. | |
| if ! grep -q 'storage: 500Gi' /tmp/storage-rendered.yaml; then | |
| echo "::error::k8s/storage/ no longer renders 'storage: 500Gi'; the size rewrite below would silently no-op" | |
| exit 1 | |
| fi | |
| sed -e 's|storageClassName: cinder-csi|storageClassName: standard|g' \ | |
| -e 's|storage: 500Gi|storage: 8Gi|' \ | |
| /tmp/storage-rendered.yaml > /tmp/storage.yaml | |
| # The node CIDR in networkpolicy.yaml ships as RFC 5737 TEST-NET-1 and | |
| # is documented there as a placeholder to be set before the first | |
| # deploy. It is not cosmetic here: kindnetd runs kube-network-policies | |
| # unconditionally, the namespace default-deny therefore applies, and | |
| # the in-tree `nfs:` PVs this chart emits are mounted by the KUBELET | |
| # from the node's own address in the host netns - matching no | |
| # podSelector and no ipBlock. Left alone, every mount on a node not | |
| # running Ganesha hangs, which the worker's DoNotSchedule spread | |
| # guarantees will happen to exactly one replica. | |
| if ! grep -q 'cidr: 192.0.2.0/24' /tmp/storage.yaml; then | |
| echo "::error::k8s/storage/networkpolicy.yaml no longer carries the 192.0.2.0/24 placeholder; this rewrite would silently no-op and every cross-node mount would hang" | |
| exit 1 | |
| fi | |
| # Derived, not hardcoded: kind pins only its IPv6 subnet, so the IPv4 | |
| # one comes from Docker's default address pool and is not 172.18/16 | |
| # by contract. | |
| NODE_CIDR=$(docker network inspect kind \ | |
| -f '{{range .IPAM.Config}}{{.Subnet}} {{end}}' \ | |
| | tr ' ' '\n' | grep -v ':' | grep -E '^[0-9.]+/[0-9]+$' | head -n1) | |
| if [ -z "$NODE_CIDR" ]; then | |
| echo "::error::could not read an IPv4 subnet from the kind docker network" | |
| docker network inspect kind -f '{{json .IPAM.Config}}' | |
| exit 1 | |
| fi | |
| echo "admitting kind nodes on 2049 from $NODE_CIDR" | |
| sed -i "s|cidr: 192.0.2.0/24|cidr: ${NODE_CIDR}|" /tmp/storage.yaml | |
| # kubeconform validated /tmp/storage-rendered.yaml, not the file that | |
| # is actually applied. Re-check the one value the seds above compute | |
| # rather than copy, so a malformed CIDR fails here and names itself | |
| # instead of surfacing as an API-server rejection mid-retry-loop. | |
| if ! grep -Eq '^[[:space:]]*cidr: [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$' /tmp/storage.yaml; then | |
| echo "::error::the rewritten NetworkPolicy CIDR is not a bare IPv4 block" | |
| grep -n 'cidr:' /tmp/storage.yaml | |
| exit 1 | |
| fi | |
| applied=0 | |
| for i in 1 2 3 4 5; do | |
| if kubectl apply -f /tmp/storage.yaml; then | |
| applied=1 | |
| echo "Storage apply succeeded on attempt $i" | |
| break | |
| fi | |
| echo "Attempt $i failed, retrying in ${i}0s..." | |
| sleep "${i}0" | |
| done | |
| # `if ... then break; fi` never trips `set -e`, so without this the | |
| # loop fell through on total failure and the step died on the bare | |
| # NotFound from the substitution below - taking the authored error | |
| # message and its diagnostic dump with it, in precisely the case the | |
| # retry loop exists to report. | |
| if [ "$applied" -ne 1 ]; then | |
| echo "::error::could not apply k8s/storage/ after 5 attempts" | |
| kubectl get all -n template-app-storage || true | |
| exit 1 | |
| fi | |
| # `kubectl wait --all` reports success against zero objects, so the | |
| # server has to be found before it is waited on. `|| true` because an | |
| # empty result must reach the explicit check below rather than kill | |
| # the step through `bash -e`. | |
| WORKLOADS=$(kubectl get deploy,statefulset -n template-app-storage -o name 2>/dev/null || true) | |
| if [ -z "$WORKLOADS" ]; then | |
| echo "::error::k8s/storage/ created no Deployment or StatefulSet in template-app-storage" | |
| kubectl get all -n template-app-storage | |
| exit 1 | |
| fi | |
| for w in $WORKLOADS; do | |
| kubectl rollout status -n template-app-storage "$w" --timeout=300s | |
| done | |
| kubectl get pods,pvc -n template-app-storage -o wide | |
| kubectl get storageclass | |
| # Diagnostic, never a gate: this is the filesystem nfs-provisioner | |
| # measures when it decides whether a workspace claim fits. | |
| kubectl exec -n template-app-storage \ | |
| "$(kubectl get pod -n template-app-storage -l app=nfs-server -o name | head -n1)" \ | |
| -- df -h /export || true | |
| - name: Deploy with Kustomize (full manifests, no filter) | |
| run: | | |
| # k8s/overlays/ci/, not prod, for the same reason as the nginx job: | |
| # prod's rq-worker requests 16Gi / 4 cpu with requests == limits and | |
| # this runner has 4 vCPU and ~15.6Gi in total, so neither replica | |
| # could ever be scheduled and both would sit Pending until "Verify | |
| # all deployments are available" failed at 180s. The CI overlay is | |
| # prod with the worker's size patched down and nothing else changed, | |
| # so the IngressRoute rendered below is prod's, unmodified. | |
| # | |
| # No cinder-csi rewrite here any more. The only Cinder-classed | |
| # object left in the tree is Ganesha's own backing claim in | |
| # k8s/storage/, and the storage step above rewrites that one - loudly | |
| # if the string ever disappears. A sed kept here would match nothing | |
| # and quietly imply the storage path was still being exercised. | |
| kubectl kustomize k8s/overlays/ci/ | \ | |
| sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' > /tmp/manifests.yaml | |
| # Size the workspaces claim for the runner's disk. nfs-provisioner | |
| # statfs()es /export on every provision and refuses outright any | |
| # claim larger than the free space it finds there - the check is | |
| # unconditional, not gated on quotas. In kind the export is a | |
| # local-path volume on the GitHub runner's disk, tens of GiB, so the | |
| # production figure could never bind and every pod would sit Pending | |
| # behind it. Fail loudly if the request has gone, rather than letting | |
| # the rewrite no-op and take the storage path out of CI with it. | |
| if ! grep -qE 'storage: [0-9]+Gi' /tmp/manifests.yaml; then | |
| echo "::error::the rendered overlay has no 'storage: <N>Gi' request; the rewrite below would silently no-op" | |
| exit 1 | |
| fi | |
| # 1Gi, not 4Gi: nfs-provisioner statfs()es /export and refuses any | |
| # claim larger than the free space it reports, and /export here is a | |
| # local-path directory on a runner disk that has just absorbed the | |
| # app image into both kind nodes' containerd. No assertion in this | |
| # suite writes anywhere near a gigabyte - the largest is ~60 short | |
| # log lines - so the headroom buys nothing and costs a 300s Bound | |
| # timeout whose message is about bytes. | |
| sed -i -E 's|storage: [0-9]+Gi|storage: 1Gi|g' /tmp/manifests.yaml | |
| for i in 1 2 3 4 5; do | |
| if kubectl apply -f /tmp/manifests.yaml; then | |
| echo "Deploy succeeded on attempt $i" | |
| break | |
| fi | |
| echo "Attempt $i failed, retrying in ${i}0s..." | |
| sleep "${i}0" | |
| done | |
| - name: Discover overlay identity | |
| # Read off prod, even though ci/ is what was applied: the CI overlay | |
| # has no identity of its own - no commonLabels, no IngressRoute patch - | |
| # it inherits prod whole and only repatches the worker's resources. | |
| # Reading the source of that identity keeps this working if the CI | |
| # overlay ever grows a second patch. | |
| run: | | |
| SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) | |
| TRAEFIK_HOSTS=$(kubectl kustomize k8s/overlays/prod/ \ | |
| | yq 'select(.kind == "IngressRoute") | .spec.routes[0].match' \ | |
| | grep -oP "Host\(\`\K[^\`]+" | tr '\n' ' ') | |
| # This shell has no pipefail, so a grep that matches nothing yields an | |
| # empty string with exit 0 - and "Curl both hostnames via Traefik" | |
| # below, the ONLY place this job exercises the app through Traefik, | |
| # then iterates zero times and passes. Unlike the nginx job, which | |
| # hardcodes its two hosts, this gate is entirely derived, so the | |
| # emptiness has to be an error rather than a quiet no-op. | |
| if [ -z "${TRAEFIK_HOSTS// /}" ]; then | |
| echo "::error::no Host() found in the prod IngressRoute match expression; the curl gate below would test nothing" | |
| kubectl kustomize k8s/overlays/prod/ | yq 'select(.kind == "IngressRoute")' | |
| exit 1 | |
| fi | |
| echo "SLUG=$SLUG" >> "$GITHUB_ENV" | |
| echo "TRAEFIK_HOSTS=$TRAEFIK_HOSTS" >> "$GITHUB_ENV" | |
| - name: Verify the workspace StorageClass contract | |
| # k8s/base/workspace-pvc.yaml names a class k8s/storage/ has to publish, | |
| # and the mount options ride on that class because a PVC has nowhere to | |
| # carry them. Checking all of it here turns drift into one failed step | |
| # with the actual values printed, instead of a Pending pod at 3am. | |
| run: | | |
| # Read the claim off the Deployment that mounts it, so namePrefix and | |
| # any second PVC in the namespace stay irrelevant. | |
| CLAIM=$(kubectl get deployment -n openms -l app=${SLUG},component=streamlit \ | |
| -o jsonpath='{.items[0].spec.template.spec.volumes[?(@.name=="workspaces")].persistentVolumeClaim.claimName}') | |
| if [ -z "${CLAIM}" ]; then | |
| echo "::error::no streamlit Deployment in openms mounts a 'workspaces' volume - there is no claim to check" | |
| kubectl get deployment -n openms -o wide | |
| exit 1 | |
| fi | |
| kubectl wait -n openms --for=jsonpath='{.status.phase}'=Bound \ | |
| "pvc/${CLAIM}" --timeout=300s | |
| MODES=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.status.accessModes[*]}') | |
| case " ${MODES} " in | |
| *" ReadWriteMany "*) ;; | |
| *) | |
| echo "::error::${CLAIM} bound as '${MODES}', not ReadWriteMany; every pod that mounts it is back to sharing one node" | |
| exit 1 | |
| ;; | |
| esac | |
| CLASS=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.spec.storageClassName}') | |
| echo "${CLAIM} is Bound ReadWriteMany on StorageClass '${CLASS}'" | |
| if ! kubectl get storageclass "${CLASS}" >/dev/null 2>&1; then | |
| echo "::error::the workspaces PVC names StorageClass '${CLASS}', which k8s/storage/ does not publish" | |
| kubectl get storageclass | |
| exit 1 | |
| fi | |
| OPTS=$(kubectl get storageclass "${CLASS}" -o jsonpath='{.mountOptions[*]}') | |
| echo "mount options: ${OPTS:-<none>}" | |
| case " ${OPTS} " in | |
| *" soft "*) | |
| echo "::error::'${CLASS}' mounts soft; a routine Ganesha restart would then truncate in-flight writes instead of blocking" | |
| exit 1 | |
| ;; | |
| *" hard "*) ;; | |
| *) | |
| echo "::error::'${CLASS}' does not pin 'hard'. It is also the kernel default, but the design states it explicitly so that adding 'soft' is a visible diff" | |
| exit 1 | |
| ;; | |
| esac | |
| echo "${OPTS}" | grep -qE '(^| )(nfs)?vers=4\.1( |$)' || { | |
| echo "::error::'${CLASS}' does not pin NFSv4.1, which is what supplies integrated locking on one port" | |
| exit 1 | |
| } | |
| echo "${OPTS}" | grep -qE '(^| )nconnect=4( |$)' || { | |
| echo "::error::'${CLASS}' does not set nconnect=4" | |
| exit 1 | |
| } | |
| if echo "${OPTS}" | grep -qE '(^| )(actimeo|acregmin|acregmax|acdirmin|acdirmax)='; then | |
| echo "::error::'${CLASS}' overrides the attribute cache timeouts; close-to-open consistency already covers this app, so actimeo stays at its default" | |
| exit 1 | |
| fi | |
| - name: Wait for Redis to be ready | |
| run: | | |
| kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s | |
| - name: Verify all deployments are available | |
| # No `|| true`: a Pending, CrashLooping or unschedulable Deployment has to | |
| # fail the job, otherwise the ingress curl below is the only real gate. | |
| # Diagnostics come from "Dump cluster state on failure". | |
| run: | | |
| kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s | |
| kubectl get pods -n openms -l app=${SLUG} -o wide | |
| kubectl get services -n openms -l app=${SLUG} | |
| - name: Curl both hostnames via Traefik | |
| run: | | |
| kubectl -n traefik port-forward svc/traefik 8080:80 & | |
| PF_PID=$! | |
| trap 'kill "$PF_PID" 2>/dev/null || true' EXIT | |
| FIRST_HOST=$(echo ${TRAEFIK_HOSTS} | awk '{print $1}') | |
| for i in $(seq 1 30); do | |
| sleep 2 | |
| if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8080/_stcore/health -H "Host: ${FIRST_HOST}"; then | |
| break | |
| fi | |
| echo "port-forward / app not ready yet, retry $i" | |
| done | |
| curled=0 | |
| for host in ${TRAEFIK_HOSTS}; do | |
| curl -fsS --resolve "$host:8080:127.0.0.1" "http://$host:8080/_stcore/health" | |
| echo "" | |
| echo "$host -> 200 OK" | |
| curled=$((curled + 1)) | |
| done | |
| # A loop that ran zero times is otherwise indistinguishable from a | |
| # loop where every host answered 200. | |
| if [ "$curled" -lt 2 ]; then | |
| echo "::error::curled $curled host(s) through Traefik, expected 2" | |
| exit 1 | |
| fi | |
| # Everything below is `.github/scripts/ci-assertions.sh`. Each assertion | |
| # discovers the namespace, the app label, the workspace claim and the | |
| # helper image from the cluster, so SLUG above is the only input any of | |
| # them needs. | |
| # | |
| # The NetworkPolicy assertion is first, and then the two placement ones, | |
| # because all three are seconds long where the storage ones are up to 45 | |
| # minutes of deliberate timeouts. They read the state that "Verify all | |
| # deployments are available" has just established, and if the workers are | |
| # not where they should be then everything after this is noise measured | |
| # against a broken deployment. | |
| - name: Assert every node is admitted to the NFS share | |
| # First, because it is the failure that disguises itself as all the | |
| # others. The share is behind a namespace-wide default-deny, the | |
| # in-tree `nfs:` PVs are mounted by the KUBELET from the node address, | |
| # and the CIDR that admits them is rewritten by the storage deploy step | |
| # rather than shipped - so if that rewrite ever no-ops, every mount on | |
| # every node except Ganesha's own hangs, and the first thing anyone | |
| # sees is an unrelated assertion timing out forty minutes later. | |
| timeout-minutes: 2 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_netpol_admits_every_node | |
| - name: Assert the worker replicas are spread across nodes | |
| # Step 4's acceptance criterion for the workers. Every replica Running | |
| # - so an unschedulable one fails here rather than being left out of | |
| # the node count - on more than one node, inside the maxSkew the | |
| # Deployment itself declares, under a spread constraint whose | |
| # labelSelector really selects these workers. | |
| # | |
| # assert_two_pods_two_nodes below does NOT cover this: it starts its | |
| # own helper pods with an explicit nodeName, which proves the volume is | |
| # reachable from two nodes and says nothing about where the scheduler | |
| # put the rq-workers. Deleting the nodeSelector is necessary for this | |
| # to pass and is not sufficient - a spread constraint scoped to the | |
| # wrong labels, or dropped entirely, fails right here. | |
| timeout-minutes: 5 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_workers_spread_across_nodes | |
| - name: Assert the workers are Guaranteed QoS | |
| # requests == limits, read off `.status.qosClass` on the RUNNING pods. | |
| # A Burstable worker scores oom_score_adj ~969 and sits near the top of | |
| # the node's kill list holding hours of work; the regression that | |
| # produces it is a one-line edit to a memory-tier component and is | |
| # otherwise invisible until something OOMKills under load. | |
| timeout-minutes: 5 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_worker_qos_guaranteed | |
| - name: Assert cross-node write visibility | |
| # Two pods holding the same claim on two different nodes at the same | |
| # time, each reading what the other wrote. | |
| timeout-minutes: 10 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_cross_node_write_visible | |
| - name: Assert the POSIX contract holds on the shared volume | |
| # Absolute symlinks, atomic rename and flock - what src/ already | |
| # depends on and what NFS is allowed to take away. It first checks the | |
| # mount really is NFS, because every one of those passes against the | |
| # node's local filesystem too, and the flock half runs across two pods | |
| # on two nodes, because a lock that only excludes a second holder on | |
| # the same node is a lock the two workers do not have. | |
| timeout-minutes: 15 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_posix_contract | |
| - name: Assert a workflow survives an NFS server restart | |
| # The single most valuable assertion in the set: delete the Ganesha pod | |
| # while a workflow is running and assert the workflow COMPLETES, with | |
| # every line of its output present and in order. A Ganesha restart is | |
| # routine, and this is exactly the case a `soft` mount would have | |
| # corrupted - into a truncated featureXML rather than a failed job. | |
| # Slow by construction: the restart and the ~90s NFSv4.1 grace period | |
| # both pass underneath a running job, which is the point. | |
| timeout-minutes: 20 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_survives_nfs_restart | |
| - name: Assert stable identity across an NFS server restart | |
| # Invariant 2. Deletes the Ganesha pod under a live client, so it is | |
| # slow by construction: NFSv4.1 spends ~90s in its grace period. | |
| timeout-minutes: 15 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_stable_identity_across_restart | |
| - name: Assert two pods on two nodes share the workspace volume | |
| # The acceptance criterion for the whole exercise, and the last of these | |
| # to go green: it stayed red until the memory-tier nodeSelector patches | |
| # were deleted. Everything above it is about the export itself and | |
| # passes as soon as ReadWriteMany works, which is what kept "RWX proven | |
| # while still single-node" observable. | |
| # | |
| # It carried `continue-on-error: true` while it was red by design - a | |
| # step that always fails makes every genuine regression around it | |
| # indistinguishable from the expected red, because nobody looks at an | |
| # already-failing job. That commit has landed, so the criterion gates | |
| # like everything else here. | |
| timeout-minutes: 10 | |
| run: | | |
| source .github/scripts/ci-assertions.sh | |
| assert_two_pods_two_nodes | |
| - name: Dump cluster state on failure | |
| if: failure() | |
| run: | | |
| echo "=== nodes ===" | |
| kubectl get nodes -o wide || true | |
| echo "=== pods (all namespaces) ===" | |
| kubectl get pods -A -o wide || true | |
| echo "=== storage tier ===" | |
| kubectl get all,pvc -n template-app-storage -o wide || true | |
| kubectl describe pods -n template-app-storage || true | |
| for p in $(kubectl get pods -n template-app-storage -o name 2>/dev/null); do | |
| kubectl logs -n template-app-storage "$p" --all-containers --prefix --tail=200 || true | |
| done | |
| echo "=== storage classes and volumes ===" | |
| kubectl get storageclass || true | |
| kubectl get pv,pvc -A || true | |
| # Unfiltered on purpose. SLUG is exported by "Discover overlay | |
| # identity", which runs AFTER the storage steps - so any failure | |
| # before it left ${SLUG} empty here, every `-l app=` matched nothing, | |
| # and this dump reported an empty namespace on a cluster that simply | |
| # had not been deployed to yet. In CI the openms namespace holds only | |
| # this app, so dropping the selector cannot lose anything. | |
| echo "=== app pods describe ===" | |
| kubectl describe pod -n openms || true | |
| echo "=== app pod logs ===" | |
| for p in $(kubectl get pods -n openms -o name 2>/dev/null); do | |
| kubectl logs -n openms "$p" --all-containers --prefix --tail=200 || true | |
| done | |
| echo "=== app pod previous logs (if crashed) ===" | |
| for p in $(kubectl get pods -n openms -o name 2>/dev/null); do | |
| kubectl logs -n openms "$p" --all-containers --prefix --tail=200 --previous || true | |
| done | |
| echo "=== traefik ingressroute ===" | |
| kubectl get ingressroute -A -o yaml || true | |
| echo "=== services + endpoints ===" | |
| kubectl get svc,endpoints -n openms || true | |
| echo "=== traefik controller logs ===" | |
| kubectl logs -n traefik -l app.kubernetes.io/name=traefik --tail=200 || true |