add build-push-deploy

.github/workflows/build-push-deploy.yml

@@ -0,0 +1,177 @@
+on:
+  workflow_call:
+    inputs:
+      service-name:
+        description: "Name of the service to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used."
+        type: string
+        required: true
+      stage-name:
+        description: "The backend environment we are building for (API calls are pointed to).  This should be one of (development, staging, production)."
+        type: string
+        required: true
+      deploy-namespace:
+        description: "The Kubernetes namespace to deploy the service to. Disables getting deploy envs from PR labels."
+        type: string
+        required: false
+      docker-build-args:
+        description: "Extra args passed to 'docker build'."
+        type: string
+        required: false
+      docker-image-ref:
+        description: "The version number or sha used in creating image tag."
+        type: string
+        required: false
+        default: "${{ github.sha }}"
+      dockerfiles:
+        description: "JSON list of dockerfiles to build, e.g. ['Dockerfile1', 'Dockerfile2']"
+        type: string
+        required: false
+        default: "['Dockerfile']"
+      docker-bake-target:
+        description: "The target to build with docker bake."
+        type: string
+        required: false
+      docker-bake-platforms:
+        description: "The platforms to build with docker bake."
+        type: string
+        required: false
+      deploy:
+        description: "Whether to deploy after building. Set to false to build and push only."
+        type: boolean
+        required: false
+        default: true
+      run-migrations:
+        description: "Whether to run the migration job before deploying."
+        type: boolean
+        required: false
+        default: false
+      migration-job-file:
+        description: "The file path to the migration k8s job YAML."
+        type: string
+        required: false
+        default: "deployment/migration-job.yaml"
+
+concurrency:
+  group: ${{ github.workflow_ref }} # workflow_ref contains the workflow name and branch ref
+  cancel-in-progress: true          # Cancel any in-progress runs on this branch - this one is newer
+
+
+jobs:
+  # Looks for PR labels like "deploy-to-<env>" so we can deploy to those envs
+  get-deploy-labels:
+    name: Get Deploy Envs
+    runs-on: mdb-dev
+    outputs:
+      deploy-envs: ${{ steps.override.outputs.deploy-envs || steps.get-labels.outputs.deploy-envs }}
+    steps:
+      - id: get-labels
+        if: inputs.deploy-namespace == ''
+        uses: mindsdb/github-actions/get-deploy-labels@main
+      - id: override
+        if: inputs.deploy-namespace != ''
+        run: echo 'deploy-envs=["${{ inputs.deploy-namespace }}"]' >> $GITHUB_OUTPUT
+
+
+  # Build docker image(s) based on Dockerfile(s) and push to ECR
+  build:
+    runs-on: mdb-dev
+    needs: [get-deploy-labels]
+    if: ${{ !inputs.docker-bake && (!inputs.deploy || needs.get-deploy-labels.outputs.deploy-envs != '[]') }}
+    strategy:
+      matrix:
+        dockerfile: ${{fromJson(inputs.dockerfiles)}}
+    env:
+      AWS_REGION: us-east-1
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.docker-image-ref }}
+      # Build via docker-bake if a bakefile is specified
+      - if: ${{ contains(matrix.dockerfile, '.hcl') }}
+        uses: mindsdb/github-actions/docker-bake@main
+        with:
+          git-sha: ${{ inputs.docker-image-ref }}
+          target: ${{ inputs.docker-bake-target }}
+          platforms: ${{ inputs.docker-bake-platforms }}
+          push-cache: false
+      # Otherwise build via regular docker
+      - if: ${{ !contains(matrix.dockerfile, '.hcl') }}
+        uses: mindsdb/github-actions/build-push-ecr@main
+        with:
+          module-name: ${{ inputs.service-name }}
+          build-for-environment: ${{ inputs.stage-name }}
+          image-ref: ${{ inputs.docker-image-ref }}
+          extra-build-args: "-f ${{ matrix.dockerfile }}"
+
+
+  migrate:
+    runs-on: mdb-dev
+    if: inputs.deploy && inputs.run-migrations
+    needs: [get-deploy-labels, build]
+    strategy:
+      matrix:
+        deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.docker-image-ref }}
+      - name: Migrate
+        run: |
+          export NAMESPACE=${{ matrix.deploy-env }}
+          export IMAGE_TAG=${{ inputs.stage-name }}-${{ inputs.docker-image-ref }}
+          export JOB_NAME=$(grep -E '^ *name:' ${{ inputs.migration-job-file }} | head -1 | awk '{print $2}')
+
+          kubectl -n $NAMESPACE delete job --ignore-not-found $JOB_NAME
+          envsubst '${IMAGE_TAG} ${NAMESPACE}' < ${{ inputs.migration-job-file }} | kubectl apply -f -
+
+          kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=1m "job/$JOB_NAME"
+
+  # Deploy the built image to the specified environments
+  # Deploys to all environments at once
+  deploy:
+    runs-on: mdb-dev
+    needs: [ get-deploy-labels, build, migrate ]
+    if: ${{ always() && !failure() && !cancelled() && inputs.deploy }}
+    strategy:
+      matrix:
+        deploy-env: ${{fromJson(needs.get-deploy-labels.outputs.deploy-envs)}}
+    environment:
+      # Assuming that ENV_URL is set in a github environment in the repo
+      # If not the link in the slack message will be borked, thats all
+      name: ${{ matrix.deploy-env }}
+      url: ${{ vars.ENV_URL }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: mindsdb/github-actions/setup-env@main
+      - name: Notify of deployment starting
+        # This same message will be updated later with the deployment status
+        id: slack
+        uses: mindsdb/github-actions/slack-deploy-msg@main
+        with:
+          channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }}
+          status: "started"
+          color: "#0099CC"
+          env-name: ${{ matrix.deploy-env }}
+          env-url: ${{ vars.ENV_URL }}
+          slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }}
+      - uses: DevOps-Nirvana/aws-helm-multi-deploy-nodocker@v6
+        # Do the actual deployment
+        with:
+          environment-slug: ${{matrix.deploy-env}}
+          k8s-namespace: ${{ matrix.deploy-env }}
+          image-tag: ${{ inputs.stage-name }}-${{ github.sha }}
+          timeout: 600s
+          # We need to wait till deployment is finished here, since the calling workflow might test the deployment env once this job is done
+          wait: "true"
+      - name: Notify of deployment finish
+        # Update the slack message from before with the deployment status
+        uses: mindsdb/github-actions/slack-deploy-msg@main
+        if: always()
+        with:
+          channel-id: ${{ secrets.SLACK_DEPLOYMENTS_CHANNEL_ID }}
+          status: "${{ job.status == 'success' && 'finished' || 'failed' }}"
+          color: "${{ job.status == 'success' && '#00C851' || '#FF4444' }}"
+          env-name: ${{ matrix.deploy-env }}
+          env-url: ${{ vars.ENV_URL }}
+          slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }}
+          update-message-id: ${{ steps.slack.outputs.ts }}

build-push-ecr/action.yml

@@ -1,5 +1,10 @@
 # Builds a docker image, then tags it with the github sha and pushes it to our Amazon ECR registry
 
+outputs:
+  image:
+    description: "Full ECR image URI with tag (e.g. 123456.dkr.ecr.us-east-1.amazonaws.com/my-service:production-abc123)"
+    value: ${{ steps.push.outputs.image }}
+
 inputs:
   module-name:
     description: "Name of the module to build. Used as the default image name and src dir unless 'image-name' or 'src-path' are used."
@@ -30,28 +35,26 @@ runs:
       uses: aws-actions/amazon-ecr-login@v2
     - shell: bash
       run: |
-        # Env var parsing
-
         INPUT_SRC_PATH=${{ inputs.src-path }}
         SRC_PATH=${INPUT_SRC_PATH:-"./"}
         INPUT_IMAGE_REF=${{ inputs.image-ref }}
         IMAGE_REF=${INPUT_IMAGE_REF:-$CI_SHA}
         IMAGE_NAME=${{ inputs.module-name }}
         REPO_IMAGE=${{ steps.login-ecr.outputs.registry }}/$IMAGE_NAME
-        DOCKER_BUILDKIT=1
         ENVIRONMENT=${{ inputs.build-for-environment }}
-        BRANCH_NAME=${{env.ENV_NAME}}
         IMAGE_TAG=$ENVIRONMENT-$IMAGE_REF
 
-        # Create repo if needed
+        # Create repo if needed (ignore error if it already exists)
         aws ecr create-repository --repository-name $IMAGE_NAME && \
-        aws ecr set-repository-policy --repository-name $IMAGE_NAME --policy-text "$(cat ${{ github.action_path }}/shared-ecr-policy.json)" || \
-        true # Just let this fail if the repo already exists
+        aws ecr set-repository-policy --repository-name $IMAGE_NAME --policy-text "$(cat ${{ github.action_path }}/shared-ecr-policy.json)" || true
 
-        docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist)
+        docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true
 
         cd $SRC_PATH
         BUILD_ARGS="--build-arg BUILD_FOR_ENVIRONMENT=$ENVIRONMENT --build-arg IMAGE_TAG=$IMAGE_TAG"
-
-        # Finally, build our runner container
         docker buildx build ${{ inputs.extra-build-args }} $BUILD_ARGS -t $REPO_IMAGE:$IMAGE_TAG -t $REPO_IMAGE:latest --push .
+
+        echo "BUILT_IMAGE=$REPO_IMAGE:$IMAGE_TAG" >> "$GITHUB_ENV"
+    - id: push
+      shell: bash
+      run: echo "image=$BUILT_IMAGE" >> "$GITHUB_OUTPUT"