From f75cbad16068d3422fa203094506c340438d0f1a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 27 Jun 2025 15:41:23 -0500 Subject: [PATCH 01/31] add prepare conda action to achieve lock within PR --- prepare-conda-env/action.yml | 78 ++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 prepare-conda-env/action.yml diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml new file mode 100644 index 00000000..9c66b79a --- /dev/null +++ b/prepare-conda-env/action.yml @@ -0,0 +1,78 @@ +name: Create Locked Conda Environment +inputs: + dependency-key: + required: true + type: string + description: | + The key of the dependency to generate, such as cpp, python, etc. + arch: + required: true + type: string + description: | + The architecture to build for, such as x86_64, aarch64, etc. + cuda-version: + required: true + type: string + description: | + The CUDA version to build for, such as 11.7, 12.1, etc. + relock: + type: boolean + default: false + description: | + Whether to force a relock of the environment. +outputs: + env-path: + description: | + The path to the conda environment. + value: ${{ steps.create-env.outputs.env-path }} + +runs: + using: 'composite' + steps: + - name: compute lockfile filename + id: lockfile-filename + shell: bash + run: | + echo "filename=.conda-lock-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}.yml" >> $GITHUB_OUTPUT + + - name: Retrieve any existing lockfile + uses: actions/cache@v4 + with: + key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ github.event.pull_request.number }} + path: ${{ steps.lockfile-filename.outputs.filename }} + restore-keys: | + conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ github.event.pull_request.number }} + + - name: Run conda-lock if needed + shell: bash + if: ${{ !isfile(steps.lockfile-filename.outputs.filename) || inputs.relock == true }} + run: | + CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) + + rapids-logger "Generate ${inputs.dependency-key} testing dependencies" + + ENV_YAML_DIR="$(mktemp -d)" + + rapids-dependency-file-generator \ + --output conda \ + --file-key test_${{ inputs.dependency-key }} \ + --prepend-channel "${CPP_CHANNEL}" \ + --matrix "cuda=${${{ inputs.cuda-version }}%.*};arch=${inputs.arch}" | tee "${ENV_YAML_DIR}/env.yaml" + + mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run --quiet | tee ${{ steps.lockfile-filename.outputs.filename }} + + - name: Create environment + shell: bash + run: | + rapids-mamba-retry env create --yes -f ${{ steps.lockfile-filename.outputs.filename }} -n test + + RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} + RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ + mkdir -p "${RAPIDS_TESTS_DIR}" + echo "RESULTS_DIR=${RESULTS_DIR}" >> $GITHUB_ENV + echo "RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR}" >> $GITHUB_ENV + + rapids-print-env + + rapids-logger "Check GPU usage" + nvidia-smi From 130d0b5bd22975e5706fd5d91b217f63c6269fce Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 17:36:57 -0500 Subject: [PATCH 02/31] if statement syntax --- prepare-conda-env/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 9c66b79a..f403589e 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -45,7 +45,7 @@ runs: - name: Run conda-lock if needed shell: bash - if: ${{ !isfile(steps.lockfile-filename.outputs.filename) || inputs.relock == true }} + if: !isfile(steps.lockfile-filename.outputs.filename) || inputs.relock == true run: | CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) From 0f31021458ffdfdea3070e81587c5cc28c50e379 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 17:46:38 -0500 Subject: [PATCH 03/31] move conditional into shell --- prepare-conda-env/action.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index f403589e..bdf66db4 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -45,21 +45,22 @@ runs: - name: Run conda-lock if needed shell: bash - if: !isfile(steps.lockfile-filename.outputs.filename) || inputs.relock == true run: | - CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) + if [ -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then + CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) - rapids-logger "Generate ${inputs.dependency-key} testing dependencies" + rapids-logger "Generate ${inputs.dependency-key} testing dependencies" - ENV_YAML_DIR="$(mktemp -d)" + ENV_YAML_DIR="$(mktemp -d)" - rapids-dependency-file-generator \ - --output conda \ - --file-key test_${{ inputs.dependency-key }} \ - --prepend-channel "${CPP_CHANNEL}" \ - --matrix "cuda=${${{ inputs.cuda-version }}%.*};arch=${inputs.arch}" | tee "${ENV_YAML_DIR}/env.yaml" + rapids-dependency-file-generator \ + --output conda \ + --file-key test_${{ inputs.dependency-key }} \ + --prepend-channel "${CPP_CHANNEL}" \ + --matrix "cuda=${${{ inputs.cuda-version }}%.*};arch=${inputs.arch}" | tee "${ENV_YAML_DIR}/env.yaml" - mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run --quiet | tee ${{ steps.lockfile-filename.outputs.filename }} + mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run --quiet | tee ${{ steps.lockfile-filename.outputs.filename }} + fi - name: Create environment shell: bash From 42bcdaf496e762cffec786788cf9940a32b2bd7d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 17:54:40 -0500 Subject: [PATCH 04/31] invert file exists logic --- prepare-conda-env/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index bdf66db4..68dfee9e 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -46,7 +46,7 @@ runs: - name: Run conda-lock if needed shell: bash run: | - if [ -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then + if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) rapids-logger "Generate ${inputs.dependency-key} testing dependencies" From 2cfc41e6ab140758be43c822caa28eff48a31443 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 18:04:44 -0500 Subject: [PATCH 05/31] debugging --- prepare-conda-env/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 68dfee9e..c914dd7b 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -46,6 +46,7 @@ runs: - name: Run conda-lock if needed shell: bash run: | + set -Eeuxo pipefail if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) @@ -60,6 +61,8 @@ runs: --matrix "cuda=${${{ inputs.cuda-version }}%.*};arch=${inputs.arch}" | tee "${ENV_YAML_DIR}/env.yaml" mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run --quiet | tee ${{ steps.lockfile-filename.outputs.filename }} + else + rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi - name: Create environment From ef2569d30384d7cd82b7f23a8a7db20d52a287a0 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 18:20:24 -0500 Subject: [PATCH 06/31] pass pr number in from shared-workflows --- prepare-conda-env/action.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index c914dd7b..8dd67f42 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -20,6 +20,10 @@ inputs: default: false description: | Whether to force a relock of the environment. + pr-number: + type: string + description: | + The pull request number to use for the cache key. outputs: env-path: description: | @@ -38,10 +42,10 @@ runs: - name: Retrieve any existing lockfile uses: actions/cache@v4 with: - key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ github.event.pull_request.number }} + key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.pr-number }} path: ${{ steps.lockfile-filename.outputs.filename }} restore-keys: | - conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ github.event.pull_request.number }} + conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.pr-number }} - name: Run conda-lock if needed shell: bash From 874466f41a4ac1f95f0454f82d8e162f45627b51 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 18:30:55 -0500 Subject: [PATCH 07/31] archive lockfile output for reference --- prepare-conda-env/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 8dd67f42..2a647542 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -69,6 +69,12 @@ runs: rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi + - name: archive lockfile for reference + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.lockfile-filename.outputs.filename }} + path: ${{ steps.lockfile-filename.outputs.filename }} + - name: Create environment shell: bash run: | From b88a4a88861f0b516e0deacf3156410bec6e21aa Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 18:34:32 -0500 Subject: [PATCH 08/31] Add gh_token pass-in --- prepare-conda-env/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 2a647542..02d7dd9b 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -51,9 +51,10 @@ runs: shell: bash run: | set -Eeuxo pipefail - if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then - CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) + CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) + + if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then rapids-logger "Generate ${inputs.dependency-key} testing dependencies" ENV_YAML_DIR="$(mktemp -d)" From c99bd8efc47fa3bb8e8a9772ad8c9cf3c67f48ec Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 20:08:46 -0500 Subject: [PATCH 09/31] require pr number input. fix deref of dep key --- prepare-conda-env/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 02d7dd9b..61c0ea8c 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -21,6 +21,7 @@ inputs: description: | Whether to force a relock of the environment. pr-number: + required: true type: string description: | The pull request number to use for the cache key. @@ -55,7 +56,7 @@ runs: CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then - rapids-logger "Generate ${inputs.dependency-key} testing dependencies" + rapids-logger "Generate ${{ inputs.dependency-key }} testing dependencies" ENV_YAML_DIR="$(mktemp -d)" From 43ebbf4487cca17cd60cbe70a75e6035f0e70ee8 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 20:51:45 -0500 Subject: [PATCH 10/31] bad substitution --- prepare-conda-env/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 61c0ea8c..219438a3 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -64,7 +64,7 @@ runs: --output conda \ --file-key test_${{ inputs.dependency-key }} \ --prepend-channel "${CPP_CHANNEL}" \ - --matrix "cuda=${${{ inputs.cuda-version }}%.*};arch=${inputs.arch}" | tee "${ENV_YAML_DIR}/env.yaml" + --matrix "cuda=${${{ inputs.cuda-version }}%.*};arch=${{ inputs.arch }}" | tee "${ENV_YAML_DIR}/env.yaml" mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run --quiet | tee ${{ steps.lockfile-filename.outputs.filename }} else From ad54fc539042dfb656029b09c1c6761156b6cc94 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 20:56:35 -0500 Subject: [PATCH 11/31] bad substitution --- prepare-conda-env/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 219438a3..cca98833 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -59,12 +59,13 @@ runs: rapids-logger "Generate ${{ inputs.dependency-key }} testing dependencies" ENV_YAML_DIR="$(mktemp -d)" + CUDA_VERSION=${{ inputs.cuda-version }} rapids-dependency-file-generator \ --output conda \ --file-key test_${{ inputs.dependency-key }} \ --prepend-channel "${CPP_CHANNEL}" \ - --matrix "cuda=${${{ inputs.cuda-version }}%.*};arch=${{ inputs.arch }}" | tee "${ENV_YAML_DIR}/env.yaml" + --matrix "cuda=${CUDA_VERSION%.*};arch=${{ inputs.arch }}" | tee "${ENV_YAML_DIR}/env.yaml" mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run --quiet | tee ${{ steps.lockfile-filename.outputs.filename }} else From d05151f4bb8caacfe912f5d33da10100339d383a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 21:19:55 -0500 Subject: [PATCH 12/31] remove quiet in hopes of seeing env creation --- prepare-conda-env/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index cca98833..f3a8243b 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -16,7 +16,7 @@ inputs: description: | The CUDA version to build for, such as 11.7, 12.1, etc. relock: - type: boolean + type: string default: false description: | Whether to force a relock of the environment. @@ -67,7 +67,7 @@ runs: --prepend-channel "${CPP_CHANNEL}" \ --matrix "cuda=${CUDA_VERSION%.*};arch=${{ inputs.arch }}" | tee "${ENV_YAML_DIR}/env.yaml" - mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run --quiet | tee ${{ steps.lockfile-filename.outputs.filename }} + mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run | tee ${{ steps.lockfile-filename.outputs.filename }} else rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi From 6ee86e6ab04ab26c38245531e41310531a75765f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 21:40:21 -0500 Subject: [PATCH 13/31] try to get PR number to work --- prepare-conda-env/action.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index f3a8243b..1ec0d780 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -20,11 +20,7 @@ inputs: default: false description: | Whether to force a relock of the environment. - pr-number: - required: true - type: string - description: | - The pull request number to use for the cache key. + outputs: env-path: description: | @@ -40,19 +36,24 @@ runs: run: | echo "filename=.conda-lock-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}.yml" >> $GITHUB_OUTPUT + - name: Get PR number + id: pr-number + shell: bash + run: | + echo "pr-number=$(echo ${GITHUB_REF_NAME} | sed 's/.*\///')" >> $GITHUB_OUTPUT + - name: Retrieve any existing lockfile uses: actions/cache@v4 with: - key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.pr-number }} + key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ steps.pr-number.outputs.pr-number }} path: ${{ steps.lockfile-filename.outputs.filename }} restore-keys: | - conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.pr-number }} + conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ steps.pr-number.outputs.pr-number }} - name: Run conda-lock if needed shell: bash run: | set -Eeuxo pipefail - CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then From af5324cc7ef436b727800606e8e165e6b76387cc Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 22:10:36 -0500 Subject: [PATCH 14/31] try conda-lock instead of mamba dry run --- prepare-conda-env/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 1ec0d780..af314c1b 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -34,7 +34,7 @@ runs: id: lockfile-filename shell: bash run: | - echo "filename=.conda-lock-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}.yml" >> $GITHUB_OUTPUT + echo "filename=conda-lock-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}.yml" >> $GITHUB_OUTPUT - name: Get PR number id: pr-number @@ -57,6 +57,7 @@ runs: CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then + mamba install -c conda-forge conda-lock rapids-logger "Generate ${{ inputs.dependency-key }} testing dependencies" ENV_YAML_DIR="$(mktemp -d)" @@ -68,7 +69,7 @@ runs: --prepend-channel "${CPP_CHANNEL}" \ --matrix "cuda=${CUDA_VERSION%.*};arch=${{ inputs.arch }}" | tee "${ENV_YAML_DIR}/env.yaml" - mamba env create --file "${ENV_YAML_DIR}/env.yaml" --dry-run | tee ${{ steps.lockfile-filename.outputs.filename }} + conda-lock -f "${ENV_YAML_DIR}/env.yaml" --kind env --lockfile ${{ steps.lockfile-filename.outputs.filename }} -p linux-${{ inputs.arch == 'amd64' ? '64' : 'aarch64' }} else rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi From b63020b1a6b37fe95a372f58374104506aea353b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 22:16:05 -0500 Subject: [PATCH 15/31] conda arch manipulation --- prepare-conda-env/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index af314c1b..dc0264a7 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -69,7 +69,9 @@ runs: --prepend-channel "${CPP_CHANNEL}" \ --matrix "cuda=${CUDA_VERSION%.*};arch=${{ inputs.arch }}" | tee "${ENV_YAML_DIR}/env.yaml" - conda-lock -f "${ENV_YAML_DIR}/env.yaml" --kind env --lockfile ${{ steps.lockfile-filename.outputs.filename }} -p linux-${{ inputs.arch == 'amd64' ? '64' : 'aarch64' }} + CONDA_ARCH="linux-$([ ${{ inputs.arch }} = 'amd64' ] && echo '64' || echo 'aarch64' )" + + conda-lock -f "${ENV_YAML_DIR}/env.yaml" --kind env --lockfile ${{ steps.lockfile-filename.outputs.filename }} -p $CONDA_ARCH else rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi From d8f456ce3ea7343fa2b2037dead42579adaf06a7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 29 Jun 2025 22:44:13 -0500 Subject: [PATCH 16/31] lockfile rename --- prepare-conda-env/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index dc0264a7..e8865da1 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -71,7 +71,8 @@ runs: CONDA_ARCH="linux-$([ ${{ inputs.arch }} = 'amd64' ] && echo '64' || echo 'aarch64' )" - conda-lock -f "${ENV_YAML_DIR}/env.yaml" --kind env --lockfile ${{ steps.lockfile-filename.outputs.filename }} -p $CONDA_ARCH + conda-lock -f "${ENV_YAML_DIR}/env.yaml" --kind env -p $CONDA_ARCH + mv conda-${CONDA_ARCH}.lock.yml ${{ steps.lockfile-filename.outputs.filename }} else rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi From 0dddfe139c70e979fc0f959f0e1013c559397c7e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 30 Jun 2025 09:41:14 -0500 Subject: [PATCH 17/31] use constant dir for downloaded packages so that local channel stays same location --- prepare-conda-env/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index e8865da1..e9e233e8 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -54,6 +54,8 @@ runs: shell: bash run: | set -Eeuxo pipefail + export RAPIDS_UNZIP_DIR="$(pwd)/downloaded-packages" + mkdir -p $RAPIDS_UNZIP_DIR CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then From fa5977d0e172d9a9ebb8c923aec6556bff63f24b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 30 Jun 2025 15:10:39 -0500 Subject: [PATCH 18/31] list downloaded files dir --- prepare-conda-env/action.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index e9e233e8..fb3b8b1d 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -57,6 +57,7 @@ runs: export RAPIDS_UNZIP_DIR="$(pwd)/downloaded-packages" mkdir -p $RAPIDS_UNZIP_DIR CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) + ls -lRa $RAPIDS_UNZIP_DIR if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then mamba install -c conda-forge conda-lock @@ -73,12 +74,28 @@ runs: CONDA_ARCH="linux-$([ ${{ inputs.arch }} = 'amd64' ] && echo '64' || echo 'aarch64' )" - conda-lock -f "${ENV_YAML_DIR}/env.yaml" --kind env -p $CONDA_ARCH + conda-lock -f "${ENV_YAML_DIR}/env.yaml" --kind env -p $CONDA_ARCH + mv conda-${CONDA_ARCH}.lock.yml ${{ steps.lockfile-filename.outputs.filename }} else rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi + # - name: Unpin local file dependencies in lockfile + # shell: python + # run: | + # import yaml + # with open(${{ steps.lockfile-filename.outputs.filename }}, 'r') as f: + # lockfile = yaml.safe_load(f) + # local_file_pkgs = (pkg.split() for pkg in os.listdir(RAPIDS_UNZIP_DIR)) + # edited_dependencies = [] + # for pkg in lockfile['dependencies']: + + # pkg['file:'] = 'file:*' + # edited_dependencies.append(pkg) + # lockfile['dependencies'] = edited_dependencies + # with open(${{ steps.lockfile-filename.outputs.filename }}, 'w') as f: + # yaml.dump(lockfile, f) - name: archive lockfile for reference uses: actions/upload-artifact@v4 with: From 52b5f558db9d278bc37c5ed5f0eb0c2a06756253 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Jul 2025 21:00:46 -0500 Subject: [PATCH 19/31] add python to unpin local file dependencies --- prepare-conda-env/action.yml | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index fb3b8b1d..279ef67d 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -81,21 +81,25 @@ runs: rapids-logger "Skipping conda-lock as it already exists and relock is not true" fi - # - name: Unpin local file dependencies in lockfile - # shell: python - # run: | - # import yaml - # with open(${{ steps.lockfile-filename.outputs.filename }}, 'r') as f: - # lockfile = yaml.safe_load(f) - # local_file_pkgs = (pkg.split() for pkg in os.listdir(RAPIDS_UNZIP_DIR)) - # edited_dependencies = [] - # for pkg in lockfile['dependencies']: - - # pkg['file:'] = 'file:*' - # edited_dependencies.append(pkg) - # lockfile['dependencies'] = edited_dependencies - # with open(${{ steps.lockfile-filename.outputs.filename }}, 'w') as f: - # yaml.dump(lockfile, f) + - name: Unpin local file dependencies in lockfile + shell: python + run: | + import yaml + with open(${{ steps.lockfile-filename.outputs.filename }}, 'r') as f: + lockfile = yaml.safe_load(f) + local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg in os.walk(RAPIDS_UNZIP_DIR) if pkg.endswith('.conda')} + print(local_file_pkg_names) + edited_dependencies = [] + for pkg in lockfile['dependencies']: + if isinstance(pkg, str): + pkg_name = pkg.split('=', 1)[0] + if pkg_name in local_file_pkg_names: + edited_dependencies.append(f'{pkg_name}') + continue + edited_dependencies.append(pkg) + lockfile['dependencies'] = edited_dependencies + with open(${{ steps.lockfile-filename.outputs.filename }}, 'w') as f: + yaml.dump(lockfile, f) - name: archive lockfile for reference uses: actions/upload-artifact@v4 with: From 4deca4a00ec61d08eeb6f5f142d8f6e329022c76 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Jul 2025 21:12:11 -0500 Subject: [PATCH 20/31] fix quoting in Python script --- prepare-conda-env/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 279ef67d..3eca01d9 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -85,9 +85,10 @@ runs: shell: python run: | import yaml - with open(${{ steps.lockfile-filename.outputs.filename }}, 'r') as f: + import os + with open("${{ steps.lockfile-filename.outputs.filename }}", 'r') as f: lockfile = yaml.safe_load(f) - local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg in os.walk(RAPIDS_UNZIP_DIR) if pkg.endswith('.conda')} + local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg in os.walk("${{ env.RAPIDS_UNZIP_DIR }}") if pkg.endswith('.conda')} print(local_file_pkg_names) edited_dependencies = [] for pkg in lockfile['dependencies']: @@ -98,7 +99,7 @@ runs: continue edited_dependencies.append(pkg) lockfile['dependencies'] = edited_dependencies - with open(${{ steps.lockfile-filename.outputs.filename }}, 'w') as f: + with open("${{ steps.lockfile-filename.outputs.filename }}", 'w') as f: yaml.dump(lockfile, f) - name: archive lockfile for reference uses: actions/upload-artifact@v4 From 48c50f21444c44dd9b3418943e3cafebbe406844 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 10:31:50 -0500 Subject: [PATCH 21/31] adjust walk dir; debug prints --- prepare-conda-env/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 3eca01d9..f2488312 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -86,19 +86,22 @@ runs: run: | import yaml import os + from pprint import pprint with open("${{ steps.lockfile-filename.outputs.filename }}", 'r') as f: lockfile = yaml.safe_load(f) - local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg in os.walk("${{ env.RAPIDS_UNZIP_DIR }}") if pkg.endswith('.conda')} + local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg in os.walk("downloaded-packages") if pkg.endswith('.conda')} print(local_file_pkg_names) edited_dependencies = [] for pkg in lockfile['dependencies']: + print(pkg) if isinstance(pkg, str): pkg_name = pkg.split('=', 1)[0] if pkg_name in local_file_pkg_names: - edited_dependencies.append(f'{pkg_name}') + edited_dependencies.append(pkg_name) continue edited_dependencies.append(pkg) lockfile['dependencies'] = edited_dependencies + pprint(lockfile) with open("${{ steps.lockfile-filename.outputs.filename }}", 'w') as f: yaml.dump(lockfile, f) - name: archive lockfile for reference From b1f891d41a906647892e1bfa48b3ca88d6b312c2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 10:35:29 -0500 Subject: [PATCH 22/31] walk makes a list --- prepare-conda-env/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index f2488312..6b23191b 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -89,7 +89,7 @@ runs: from pprint import pprint with open("${{ steps.lockfile-filename.outputs.filename }}", 'r') as f: lockfile = yaml.safe_load(f) - local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg in os.walk("downloaded-packages") if pkg.endswith('.conda')} + local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg_list in os.walk("downloaded-packages") for pkg in pkg_list if pkg.endswith('.conda')} print(local_file_pkg_names) edited_dependencies = [] for pkg in lockfile['dependencies']: From a89c9ac1c79b2c3fafff03d4120c8f4c8f8ac40e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 10:50:26 -0500 Subject: [PATCH 23/31] preserve version pin from local file list --- prepare-conda-env/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 6b23191b..0d1d5d89 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -89,7 +89,7 @@ runs: from pprint import pprint with open("${{ steps.lockfile-filename.outputs.filename }}", 'r') as f: lockfile = yaml.safe_load(f) - local_file_pkg_names = {pkg.rsplit('-', 2)[0] for _, _, pkg_list in os.walk("downloaded-packages") for pkg in pkg_list if pkg.endswith('.conda')} + local_file_pkg_names = {pkg.rsplit('-', 2)[0]: pkg.rsplit('-', 2)[1] for _, _, pkg_list in os.walk("downloaded-packages") for pkg in pkg_list if pkg.endswith('.conda')} print(local_file_pkg_names) edited_dependencies = [] for pkg in lockfile['dependencies']: @@ -97,7 +97,7 @@ runs: if isinstance(pkg, str): pkg_name = pkg.split('=', 1)[0] if pkg_name in local_file_pkg_names: - edited_dependencies.append(pkg_name) + edited_dependencies.append(f'{pkg_name}={local_file_pkg_names[pkg_name]}') continue edited_dependencies.append(pkg) lockfile['dependencies'] = edited_dependencies From bba075d4a36785ec74ba2834d1369e84d20cf903 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 11:40:53 -0500 Subject: [PATCH 24/31] use regex to strip alpha spec from version --- prepare-conda-env/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 0d1d5d89..fadb012b 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -92,13 +92,18 @@ runs: local_file_pkg_names = {pkg.rsplit('-', 2)[0]: pkg.rsplit('-', 2)[1] for _, _, pkg_list in os.walk("downloaded-packages") for pkg in pkg_list if pkg.endswith('.conda')} print(local_file_pkg_names) edited_dependencies = [] + alpha_package_regex = re.compile(r'(.+?)\.a\d+$') for pkg in lockfile['dependencies']: print(pkg) if isinstance(pkg, str): - pkg_name = pkg.split('=', 1)[0] + pkg_name, version, build = pkg.split('=') if pkg_name in local_file_pkg_names: edited_dependencies.append(f'{pkg_name}={local_file_pkg_names[pkg_name]}') continue + elif alpha_package_regex.match(version): + version = alpha_package_regex.match(version).groups()[0] + edited_dependencies.append(f'{pkg_name}={version}') + continue edited_dependencies.append(pkg) lockfile['dependencies'] = edited_dependencies pprint(lockfile) From c8df07a36cbf9c6a28c1d442d733ffb8f73274f2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 11:42:53 -0500 Subject: [PATCH 25/31] import re --- prepare-conda-env/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index fadb012b..e4dad6a7 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -86,6 +86,7 @@ runs: run: | import yaml import os + import re from pprint import pprint with open("${{ steps.lockfile-filename.outputs.filename }}", 'r') as f: lockfile = yaml.safe_load(f) From 6bde49f0e28ac8f0e7691c63290e2b1066482cd6 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 11:50:36 -0500 Subject: [PATCH 26/31] adjust regex --- prepare-conda-env/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index e4dad6a7..282c87e6 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -93,7 +93,7 @@ runs: local_file_pkg_names = {pkg.rsplit('-', 2)[0]: pkg.rsplit('-', 2)[1] for _, _, pkg_list in os.walk("downloaded-packages") for pkg in pkg_list if pkg.endswith('.conda')} print(local_file_pkg_names) edited_dependencies = [] - alpha_package_regex = re.compile(r'(.+?)\.a\d+$') + alpha_package_regex = re.compile(r'(.+?)a\d+$') for pkg in lockfile['dependencies']: print(pkg) if isinstance(pkg, str): From 05b55bd109b059908a6d0ce3fd79ea0f4d8d64d3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 11:57:52 -0500 Subject: [PATCH 27/31] clean up and comment --- prepare-conda-env/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 282c87e6..0b6c9bd3 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -87,27 +87,27 @@ runs: import yaml import os import re - from pprint import pprint with open("${{ steps.lockfile-filename.outputs.filename }}", 'r') as f: lockfile = yaml.safe_load(f) local_file_pkg_names = {pkg.rsplit('-', 2)[0]: pkg.rsplit('-', 2)[1] for _, _, pkg_list in os.walk("downloaded-packages") for pkg in pkg_list if pkg.endswith('.conda')} - print(local_file_pkg_names) edited_dependencies = [] alpha_package_regex = re.compile(r'(.+?)a\d+$') for pkg in lockfile['dependencies']: - print(pkg) if isinstance(pkg, str): pkg_name, version, build = pkg.split('=') if pkg_name in local_file_pkg_names: + # For local packages, we need to pick up the current local files. In other words, we need to replace + # the pins in the lockfile with pins for the current local files. edited_dependencies.append(f'{pkg_name}={local_file_pkg_names[pkg_name]}') continue elif alpha_package_regex.match(version): + # Exact pins to alpha packages will break depending on the contents of the nightly builds. + # We need to unpin the alpha package to the version without the alpha, which will allow the current nightly build to be used. version = alpha_package_regex.match(version).groups()[0] edited_dependencies.append(f'{pkg_name}={version}') continue edited_dependencies.append(pkg) lockfile['dependencies'] = edited_dependencies - pprint(lockfile) with open("${{ steps.lockfile-filename.outputs.filename }}", 'w') as f: yaml.dump(lockfile, f) - name: archive lockfile for reference From bb0581973c5a1de80faa0e7af9e7608bdc87e122 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 12:24:32 -0500 Subject: [PATCH 28/31] simplify and add python to matrix where appropriate --- prepare-conda-env/action.yml | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 0b6c9bd3..82c85532 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -1,31 +1,24 @@ name: Create Locked Conda Environment +description: | + Creates a locked conda environment for the given dependency key, CUDA version, architecture, and Python version. + The environment lock is local to this PR. inputs: dependency-key: required: true - type: string description: | The key of the dependency to generate, such as cpp, python, etc. arch: required: true - type: string description: | The architecture to build for, such as x86_64, aarch64, etc. cuda-version: required: true - type: string description: | The CUDA version to build for, such as 11.7, 12.1, etc. - relock: - type: string - default: false + py-version: + default: "none" description: | - Whether to force a relock of the environment. - -outputs: - env-path: - description: | - The path to the conda environment. - value: ${{ steps.create-env.outputs.env-path }} + The Python version to build for, such as 3.10, 3.11, etc. runs: using: 'composite' @@ -34,7 +27,7 @@ runs: id: lockfile-filename shell: bash run: | - echo "filename=conda-lock-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}.yml" >> $GITHUB_OUTPUT + echo "filename=conda-lock-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}.yml" >> $GITHUB_OUTPUT - name: Get PR number id: pr-number @@ -45,10 +38,10 @@ runs: - name: Retrieve any existing lockfile uses: actions/cache@v4 with: - key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ steps.pr-number.outputs.pr-number }} + key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} path: ${{ steps.lockfile-filename.outputs.filename }} restore-keys: | - conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ steps.pr-number.outputs.pr-number }} + conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} - name: Run conda-lock if needed shell: bash @@ -59,18 +52,23 @@ runs: CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) ls -lRa $RAPIDS_UNZIP_DIR - if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ] || [ "${{ inputs.relock }}" == "true" ]; then + if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ]; then mamba install -c conda-forge conda-lock rapids-logger "Generate ${{ inputs.dependency-key }} testing dependencies" ENV_YAML_DIR="$(mktemp -d)" CUDA_VERSION=${{ inputs.cuda-version }} + MATRIX_STRING="cuda=${CUDA_VERSION%.*};arch=${{ inputs.arch }}" + if [ "${{ inputs.py-version }}" != "none" ]; then + MATRIX_STRING="${MATRIX_STRING};py=${{ inputs.py-version }}" + fi + rapids-dependency-file-generator \ --output conda \ --file-key test_${{ inputs.dependency-key }} \ --prepend-channel "${CPP_CHANNEL}" \ - --matrix "cuda=${CUDA_VERSION%.*};arch=${{ inputs.arch }}" | tee "${ENV_YAML_DIR}/env.yaml" + --matrix "${MATRIX_STRING}" | tee "${ENV_YAML_DIR}/env.yaml" CONDA_ARCH="linux-$([ ${{ inputs.arch }} = 'amd64' ] && echo '64' || echo 'aarch64' )" @@ -78,7 +76,7 @@ runs: mv conda-${CONDA_ARCH}.lock.yml ${{ steps.lockfile-filename.outputs.filename }} else - rapids-logger "Skipping conda-lock as it already exists and relock is not true" + rapids-logger "Re-using existing lockfile" fi - name: Unpin local file dependencies in lockfile From c6b2423a4d841225fb67386dd1aa6defc5a82d24 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 12:53:52 -0500 Subject: [PATCH 29/31] add github.job to cache key --- prepare-conda-env/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 82c85532..2997e37d 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -38,10 +38,10 @@ runs: - name: Retrieve any existing lockfile uses: actions/cache@v4 with: - key: conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} + key: ${{ github.job }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} path: ${{ steps.lockfile-filename.outputs.filename }} restore-keys: | - conda-lockfile-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} + ${{ github.job }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} - name: Run conda-lock if needed shell: bash From c9d3e2eb6f2552d99c3281f68aa4e9d303fa7443 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 13:14:52 -0500 Subject: [PATCH 30/31] tweak input dependency key --- prepare-conda-env/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 2997e37d..52e5df9f 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -27,7 +27,7 @@ runs: id: lockfile-filename shell: bash run: | - echo "filename=conda-lock-${{ inputs.dependency-key }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}.yml" >> $GITHUB_OUTPUT + echo "filename=${{ github.job }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}.yml" >> $GITHUB_OUTPUT - name: Get PR number id: pr-number @@ -49,12 +49,12 @@ runs: set -Eeuxo pipefail export RAPIDS_UNZIP_DIR="$(pwd)/downloaded-packages" mkdir -p $RAPIDS_UNZIP_DIR - CPP_CHANNEL=$(rapids-download-conda-from-github ${{ inputs.dependency-key }}) + CPP_CHANNEL=$(rapids-download-conda-from-github cpp) ls -lRa $RAPIDS_UNZIP_DIR if [ ! -f "${{ steps.lockfile-filename.outputs.filename }}" ]; then mamba install -c conda-forge conda-lock - rapids-logger "Generate ${{ inputs.dependency-key }} testing dependencies" + rapids-logger "Generate testing dependencies" ENV_YAML_DIR="$(mktemp -d)" CUDA_VERSION=${{ inputs.cuda-version }} From 563d8379fe2b48dd7881192b4fe4d35b47255277 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Jul 2025 14:46:39 -0500 Subject: [PATCH 31/31] use otel service name as basis for cache key --- prepare-conda-env/action.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/prepare-conda-env/action.yml b/prepare-conda-env/action.yml index 52e5df9f..1d9c5ea4 100644 --- a/prepare-conda-env/action.yml +++ b/prepare-conda-env/action.yml @@ -23,11 +23,12 @@ inputs: runs: using: 'composite' steps: - - name: compute lockfile filename - id: lockfile-filename + - uses: rapidsai/shared-actions/telemetry-impls/set-otel-service-name@main + - name: Get service name as cache key + id: cache-key shell: bash run: | - echo "filename=${{ github.job }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}.yml" >> $GITHUB_OUTPUT + echo "cache-key=$(echo ${OTEL_SERVICE_NAME} | sed 's/[\/,]/ /g; s/[[:space:]]\+/-/g; s/--*/-/g')" >> $GITHUB_OUTPUT - name: Get PR number id: pr-number @@ -35,13 +36,19 @@ runs: run: | echo "pr-number=$(echo ${GITHUB_REF_NAME} | sed 's/.*\///')" >> $GITHUB_OUTPUT + - name: Get lockfile filename + id: lockfile-filename + shell: bash + run: | + echo "filename=${{ steps.cache-key.outputs.cache-key }}-${{ steps.pr-number.outputs.pr-number }}.yml" >> $GITHUB_OUTPUT + - name: Retrieve any existing lockfile uses: actions/cache@v4 with: - key: ${{ github.job }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} + key: ${{ steps.cache-key.outputs.cache-key }}-${{ steps.pr-number.outputs.pr-number }} path: ${{ steps.lockfile-filename.outputs.filename }} restore-keys: | - ${{ github.job }}-${{ inputs.cuda-version }}-${{ inputs.arch }}-${{ inputs.py-version }}-${{ steps.pr-number.outputs.pr-number }} + ${{ steps.cache-key.outputs.cache-key }}-${{ steps.pr-number.outputs.pr-number }} - name: Run conda-lock if needed shell: bash