Refactored github workflows and scripts to allow for additional artifacts to be saved

This enables other projects to re-use the build package workflow but also include additional artifacts they need: such as configured and built core directory.

Ticket: ENT-10487
Changelog: none
(cherry picked from commit df64e02)

Author: craigcomstock

.github/workflows/build-using-buildscripts.yml

@@ -2,6 +2,13 @@ name: Build dependencies and CFEngine hub package
 
 on:
   workflow_call:
+    inputs:
+      additional_artifacts:
+        description: 'Additional files or directories to include in artifacts'
+        default: ''
+        required: false
+        type: string
+
     secrets:
       GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO:
         required: true
@@ -14,8 +21,8 @@ on:
 
 jobs:
   build_cfengine_hub_package:
-    name: Build package and run selenium tests
-    runs-on: ubuntu-20.04
+    name: Build package
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout Together Action
         uses: actions/checkout@v3
@@ -46,9 +53,10 @@ jobs:
           path: masterfiles
           ref: ${{steps.together.outputs.masterfiles || github.base_ref}}
 
-      - name: Checkout Buildscripts (current project)
+      - name: Checkout Buildscripts
         uses: actions/checkout@v3
         with:
+          repository: cfengine/buildscripts
           path: buildscripts
           fetch-depth: 20
 
@@ -88,13 +96,13 @@ jobs:
         run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
         working-directory: buildscripts/deps-packaging
 
-      - name: restore packages cache
+      - name: restore artifacts cache
         uses: actions/cache/restore@v3
         with:
-          path: packages
-          key: packages-${{ env.PACKAGE_SHA }}
+          path: artifacts
+          key: artifacts-${{ env.PACKAGE_SHA }}
           restore-keys: |
-            packages-${{ env.PACKAGE_SHA }}
+            artifacts-${{ env.PACKAGE_SHA }}
 
       - name: Restore dependency cache
         uses: actions/cache/restore@v3
@@ -106,25 +114,36 @@ jobs:
             deps-master
             deps
 
+      - name: restore configured and built core and nova projects
+        uses: actions/cache/restore@v3
+        with:
+          path: build
+          key: build-${{ env.PACKAGE_SHA }}
+          restore-keys: |
+            build-${{ env.PACKAGE_SHA }}
+
+
       - name: Build package in docker
         env:
             GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }}
         run: |
-            if [ ! -f packages/cfe*deb ]; then
-                buildscripts/ci/docker-build-package.sh
-            fi
+          if [ ! -f artifacts/cfe*deb ]; then
+              buildscripts/ci/docker-build-package.sh ${{ inputs.additional_artifacts }}
+          fi
 
       - name: Save dependency cache
         uses: actions/cache/save@v3
         with:
           path: cache
           key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
 
-      - name: Save packages cache
+      - name: Save artifacts cache
         uses: actions/cache/save@v3
         with:
-          path: packages
-          key: packages-${{ env.PACKAGE_SHA }}
+          path: |
+              artifacts
+              packages
+          key: artifacts-${{ env.PACKAGE_SHA }}
 
       - name: Save artifacts
         if: success() || failure()

.github/workflows/deployment-tests.yml

@@ -90,13 +90,13 @@ jobs:
         run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
         working-directory: buildscripts/deps-packaging
 
-      - name: restore packages cache
+      - name: restore artifacts cache
         uses: actions/cache/restore@v3
         with:
-          path: packages
-          key: packages-${{ env.PACKAGE_SHA }}
+          path: artifacts
+          key: artifacts-${{ env.PACKAGE_SHA }}
           restore-keys: |
-            packages-${{ env.PACKAGE_SHA }}
+            artifacts-${{ env.PACKAGE_SHA }}
 
       - name: Restore dependency cache
         uses: actions/cache/restore@v3
@@ -113,9 +113,9 @@ jobs:
         env:
             GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE: ${{ secrets.GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE }}
         run: |
-            if [ ! -f packages/cfe*deb ]; then
-                buildscripts/ci/docker-build-package.sh
-            fi
+          if [ ! -f artifacts/cfe*deb ]; then
+              buildscripts/ci/docker-build-package.sh
+          fi
 
       - name: Run deployment tests
         run: buildscripts/ci/docker-deployment-tests.sh
@@ -126,11 +126,11 @@ jobs:
           path: cache
           key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
 
-      - name: Save packages cache
+      - name: Save artifacts cache
         uses: actions/cache/save@v3
         with:
-          path: packages
-          key: packages-${{ env.PACKAGE_SHA }}
+          path: artifacts
+          key: artifacts-${{ env.PACKAGE_SHA }}
 
       - name: Save artifacts
         if: success() || failure()

ci/build.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 # build cfengine hub package
+# $@ -- optional space separated paths to copy to artifacts
+
 set -ex
+
+additional_artifacts="$@"
 export PROJECT=nova
 export NO_CONFIGURE=1
 export BUILD_TYPE=DEBUG
@@ -30,9 +34,15 @@ time sudo rm -rf /var/cfengine
 time sudo rm -rf /opt/cfengine
 time ./buildscripts/build-scripts/install-dependencies
 time ./buildscripts/build-scripts/package
-sudo mkdir -p packages
-sudo cp cfengine-nova-hub/*.deb packages/ || true
-sudo cp cfengine-nova-hub/*.rpm packages/ || true
+
+sudo mkdir -p artifacts
+sudo cp cfengine-nova-hub/*.deb artifacts/ || true
+sudo cp cfengine-nova-hub/*.rpm artifacts/ || true
+
+for artifact_path in $additional_artifacts; do
+  sudo cp -r "$artifact_path" artifacts/ || true
+done
+
 
 # todo maybe save the cache cp -R ~/.cache buildscripts/ci/cache
 

ci/deployment-tests.sh

@@ -27,8 +27,8 @@ function failure() {
 }
 
 if [ ! -d /var/cfengine ]; then
-  # ci and local buildscripts should place built packages in $NTECH_ROOT/packages
-  sudo dpkg -i "$NTECH_ROOT"/packages/cfengine-nova-hub*deb
+  # ci and local buildscripts should place built packages in $NTECH_ROOT/artifacts
+  sudo dpkg -i "$NTECH_ROOT"/artifacts/cfengine-nova-hub*deb
 fi
 
 

ci/docker-build-package.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 # run the build in a docker container
+# $@ -- additional artifact paths to save
+additional_artifacts="$@"
 set -ex
 
 # find the dir two levels up from here, home of all the repositories
@@ -19,7 +21,7 @@ set +x # hide secrets
 if [ -n "$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE" ]; then
   export SECRET="$GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE"
 else
-  if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/sftp-cache.sec)"; then
+  if ! export SECRET="$(pass mystiko/developers/CFEngine/jenkins/jenkins_sftp_cache@github)"; then
     echo "The sftp cache ssh secret key must be provided, either with environment variable GH_ACTIONS_SSH_KEY_BUILD_ARTIFACTS_CACHE or access to mystiko path developers/CFEngine/jenkins/sftp-cache.sec"
     exit 1
   fi
@@ -29,7 +31,7 @@ set -x # done hiding secrets
 docker run -d --env SECRET --env JOB_BASE_NAME --privileged -v "${NTECH_ROOT}":/data --name $name $name
 
 # copy local caches to docker container
-mkdir -p "${NTECH_ROOT}/packages"
+mkdir -p "${NTECH_ROOT}/artifacts"
 mkdir -p "${NTECH_ROOT}/cache"
 
 # setup host key trust
@@ -49,14 +51,14 @@ docker exec -i $name bash -c "mkdir -p ~/.ssh"
 docker exec -i $name bash -c "echo $pubkey >> ~/.ssh/known_hosts"
 
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/setup-projects.sh'
-docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'
+docker exec -i $name bash -c "cd /data; ./buildscripts/ci/build.sh ${additional_artifacts}"
 
-# save back cache and packages to host for handling by CI and such
+# save back cache and artifacts to host for handling by CI and such
 docker cp $name:/root/.cache/. "${NTECH_ROOT}/cache/"
-docker cp $name:/data/packages/. "${NTECH_ROOT}/packages/"
+docker cp $name:/data/artifacts/. "${NTECH_ROOT}/artifacts/"
 
 rc=1 # if we find no packages, fail
-for f in packages/*.deb; do
+for f in artifacts/*.deb; do
   [ -f "$f" ] && rc=0
   break
 done

ci/docker-deployment-tests.sh

@@ -21,8 +21,8 @@ if docker ps -a | grep $name; then
 fi
 docker run -d --privileged -v "${NTECH_ROOT}":/data --name $name $name || true
 
-if [ ! -d "${NTECH_ROOT}/packages" ]; then
-  echo "${NTECH_ROOT}/packages directory should exist and have a cfengine-nova-hub package there"
+if [ ! -d "${NTECH_ROOT}/artifacts" ]; then
+  echo "${NTECH_ROOT}/artifacts directory should exist and have a cfengine-nova-hub package there"
   exit 1
 fi
 docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/deployment-tests.sh'