Merge pull request #625 from BioContainers/ci/dockerfile-risk-scan #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build and publish containers merged to master, on GitHub-hosted runners. | |
| # A merge normally touches one container (enforced at PR time), but a direct push | |
| # may touch several, so `detect` produces a matrix and `publish` fans out over it. | |
| name: publish | |
| on: | |
| push: | |
| branches: [master] | |
| workflow_dispatch: | |
| inputs: | |
| container: | |
| description: "Container name (top-level directory, e.g. aragorn)" | |
| required: true | |
| version: | |
| description: "Version directory (e.g. 1.2.41)" | |
| required: true | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: publish-${{ github.sha }}-${{ github.event.inputs.container }} | |
| jobs: | |
| detect: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.list.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Determine containers to publish | |
| id: list | |
| env: | |
| IN_CONTAINER: ${{ github.event.inputs.container }} | |
| IN_VERSION: ${{ github.event.inputs.version }} | |
| EVENT_BEFORE: ${{ github.event.before }} | |
| EVENT_SHA: ${{ github.sha }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| if [ ! -f "$IN_CONTAINER/$IN_VERSION/Dockerfile" ]; then | |
| echo "No Dockerfile at $IN_CONTAINER/$IN_VERSION/Dockerfile"; exit 1 | |
| fi | |
| MATRIX=$(jq -nc --arg c "$IN_CONTAINER" --arg v "$IN_VERSION" \ | |
| '[{path:($c+"/"+$v), container:$c, version:$v}]') | |
| echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" | |
| echo "Manual publish: $IN_CONTAINER/$IN_VERSION" | |
| else | |
| # Use the push range so merge commits and multi-commit pushes are covered | |
| # (git diff-tree on a merge commit alone yields no files). | |
| if [ -z "$EVENT_BEFORE" ] || [ "$EVENT_BEFORE" = "0000000000000000000000000000000000000000" ]; then | |
| git diff --name-only "${EVENT_SHA}^" "${EVENT_SHA}" > changed_files.txt 2>/dev/null || \ | |
| git show --pretty= --name-only "${EVENT_SHA}" > changed_files.txt | |
| else | |
| git diff --name-only "$EVENT_BEFORE" "$EVENT_SHA" > changed_files.txt | |
| fi | |
| echo "Changed files:"; cat changed_files.txt | |
| python3 .github/scripts/validate.py list \ | |
| --changed-files changed_files.txt --out matrix.json | |
| fi | |
| publish: | |
| needs: detect | |
| if: ${{ needs.detect.outputs.matrix != '' && needs.detect.outputs.matrix != '[]' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| security-events: write # upload the Trivy SARIF | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.detect.outputs.matrix) }} | |
| env: | |
| C: ${{ matrix.container }} | |
| V: ${{ matrix.version }} | |
| # Singularity is uploaded to the BioContainers S3 only when these are set as | |
| # secrets; job-level so the step `if` guard can see the endpoint. | |
| S3_ENDPOINT: ${{ secrets.BIOCONTAINERS_S3_ENDPOINT }} | |
| S3_BUCKET: ${{ secrets.BIOCONTAINERS_S3_BUCKET }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| # Build each target platform locally so Trivy can gate the EXACT images that get | |
| # published. buildx caches these builds, so the later `--push` reuses the same | |
| # layers — the pushed image is byte-for-byte what was scanned. arm64 is | |
| # best-effort: many legacy base images are amd64-only, so its build fails fast. | |
| - name: Build amd64 image | |
| run: docker buildx build --load --platform linux/amd64 -t bcimg:amd64 "$C/$V" | |
| - name: Build arm64 image (best-effort) | |
| id: arm64 | |
| continue-on-error: true | |
| run: docker buildx build --load --platform linux/arm64 -t bcimg:arm64 "$C/$V" | |
| - name: Extract labels | |
| run: docker inspect --format '{{json .Config.Labels}}' bcimg:amd64 > labels.json | |
| - name: Check labels and compute tag | |
| id: check | |
| run: | | |
| python3 .github/scripts/validate.py check \ | |
| --container "$C" --version "$V" \ | |
| --labels labels.json --dockerfile "$C/$V/Dockerfile" --out report.json | |
| - name: Run tests (test-cmds.txt) | |
| run: | | |
| TESTS="$C/$V/test-cmds.txt" | |
| if [ ! -f "$TESTS" ]; then echo "No test-cmds.txt, skipping."; exit 0; fi | |
| status=0 | |
| while IFS= read -r cmd || [ -n "$cmd" ]; do | |
| cmd="${cmd%$'\r'}" # tolerate CRLF line endings | |
| case "$cmd" in ''|\#*) continue ;; esac # skip blank and comment lines | |
| echo "::group::TEST $cmd" | |
| if docker run --rm bcimg:amd64 $cmd; then echo "ok"; else echo "FAILED"; status=1; fi | |
| echo "::endgroup::" | |
| done < "$TESTS" | |
| exit $status | |
| # --- Security gate: scan every platform that WILL be published, before push. --- | |
| # A fixable HIGH/CRITICAL CVE fails the job so nothing is pushed. Unfixable OS | |
| # CVEs are ignored; a per-container `.trivyignore` (CVE IDs) can accept specific ones. | |
| - name: Prepare Trivy ignore file | |
| run: | | |
| if [ -f "$C/$V/.trivyignore" ]; then | |
| cp "$C/$V/.trivyignore" .trivyignore | |
| echo "Using per-container .trivyignore:"; cat .trivyignore | |
| fi | |
| - name: Trivy gate (amd64) | |
| uses: aquasecurity/trivy-action@0.35.0 | |
| with: | |
| image-ref: bcimg:amd64 | |
| format: sarif | |
| output: trivy.sarif | |
| severity: HIGH,CRITICAL | |
| ignore-unfixed: true | |
| exit-code: '1' | |
| - name: Upload Trivy results to Security tab | |
| if: always() | |
| continue-on-error: true | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: trivy.sarif | |
| category: trivy-${{ env.C }} | |
| - name: Trivy gate (arm64) | |
| if: ${{ steps.arm64.outcome == 'success' }} | |
| uses: aquasecurity/trivy-action@0.35.0 | |
| with: | |
| image-ref: bcimg:arm64 | |
| format: table | |
| severity: HIGH,CRITICAL | |
| ignore-unfixed: true | |
| exit-code: '1' | |
| - name: Log in to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Push scanned image(s) | |
| env: | |
| TAG: ${{ steps.check.outputs.tag }} # validated [A-Za-z0-9._-]; used only as "$TAG" | |
| ARM64_OK: ${{ steps.arm64.outcome }} | |
| run: | | |
| IMG="biocontainers/${C}:${TAG}" | |
| # --push reuses the buildx cache from the --load builds above, so the pushed | |
| # images are exactly the ones Trivy just gated. | |
| if [ "$ARM64_OK" = "success" ]; then | |
| docker buildx build --platform linux/amd64,linux/arm64 --push -t "$IMG" "$C/$V" | |
| echo "Published multi-arch (amd64+arm64, both scanned): $IMG" | |
| echo "- \`$IMG\` — **multi-arch** (amd64 + arm64, both scanned)" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| docker buildx build --platform linux/amd64 --push -t "$IMG" "$C/$V" | |
| echo "Published amd64-only (scanned): $IMG" | |
| echo "- \`$IMG\` — **amd64-only** (base has no arm64)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| # --- Singularity: build a .sif and upload to the BioContainers S3, served at | |
| # https://containers.biocontainers.pro/s3/SingImgsRepo/<tool>/<tag>/<tool>_<tag>.sif | |
| # Runs only when the S3 secrets are configured (otherwise skipped, no failure). --- | |
| # These run only when the S3 endpoint is configured, and are best-effort | |
| # (continue-on-error): the DockerHub push above is the release of record, so a | |
| # missing bucket/credential or S3 hiccup must never fail an already-published | |
| # container. Failures surface as a warning annotation. | |
| - name: Set up Apptainer (Singularity) | |
| if: ${{ env.S3_ENDPOINT != '' }} | |
| continue-on-error: true | |
| uses: eWaterCycle/setup-apptainer@v2 | |
| - name: Convert to Singularity .sif | |
| if: ${{ env.S3_ENDPOINT != '' }} | |
| continue-on-error: true | |
| env: | |
| TAG: ${{ steps.check.outputs.tag }} | |
| run: apptainer build "${C}_${TAG}.sif" "docker://biocontainers/${C}:${TAG}" | |
| - name: Upload Singularity image to BioContainers S3 | |
| if: ${{ env.S3_ENDPOINT != '' }} | |
| continue-on-error: true | |
| env: | |
| TAG: ${{ steps.check.outputs.tag }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.BIOCONTAINERS_S3_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.BIOCONTAINERS_S3_SECRET_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.BIOCONTAINERS_S3_REGION }} | |
| # The S3 store is Ceph/RGW. aws-cli v2.23+ adds default CRC32 checksums to | |
| # multipart uploads, which this RGW rejects (InvalidRequest on | |
| # CreateMultipartUpload). Revert to legacy behaviour: only checksum when the | |
| # API strictly requires it. Without this, every .sif over ~8 MB fails to upload. | |
| AWS_REQUEST_CHECKSUM_CALCULATION: when_required | |
| AWS_RESPONSE_CHECKSUM_VALIDATION: when_required | |
| run: | | |
| if [ ! -f "${C}_${TAG}.sif" ]; then echo "No .sif built, skipping upload."; exit 0; fi | |
| KEY="SingImgsRepo/${C}/${TAG}/${C}_${TAG}.sif" | |
| aws s3 cp "${C}_${TAG}.sif" "s3://${S3_BUCKET}/${KEY}" --endpoint-url "$S3_ENDPOINT" | |
| URL="https://containers.biocontainers.pro/s3/${KEY}" | |
| echo "Singularity image served at: ${URL}" | |
| echo "- Singularity: [\`${KEY}\`](${URL})" >> "$GITHUB_STEP_SUMMARY" |