A Devcontainer environment update intended to work on Caltrans Laptops with Rancher Desktop #1072
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build dbt Model Report | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - .github/workflows/build-model-report.yml | |
| - 'warehouse/**' | |
| pull_request: | |
| paths: | |
| - .github/workflows/build-model-report.yml | |
| - 'warehouse/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| PYTHON_VERSION: '3.11' | |
| SERVICE_ACCOUNT: 'github-actions-service-account@cal-itp-data-infra-staging.iam.gserviceaccount.com' | |
| WORKLOAD_IDENTITY_PROVIDER: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions/providers/data-infra' | |
| PROJECT_ID: 'cal-itp-data-infra-staging' | |
| DBT_TARGET: 'staging' | |
| DBT_ARTIFACTS_BUCKET: 'calitp-staging-dbt-artifacts' | |
| GOOGLE_CLOUD_PROJECT: 'cal-itp-data-infra-staging' | |
| jobs: | |
| compile: | |
| name: Compile dbt | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Authenticate Google Service Account | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: 'true' | |
| project_id: ${{ env.PROJECT_ID }} | |
| workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.SERVICE_ACCOUNT }} | |
| - name: Setup GCloud utilities | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| working-directory: warehouse | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| enable-cache: true | |
| cache-python: true | |
| - name: Install the project | |
| working-directory: warehouse | |
| run: uv sync --locked --all-extras --dev | |
| - name: Install dbt dependencies | |
| working-directory: warehouse | |
| run: uv run dbt deps | |
| - name: Print dbt environment | |
| working-directory: warehouse | |
| run: uv run dbt debug --target ${{ env.DBT_TARGET }} | |
| - name: Compile dbt | |
| working-directory: warehouse | |
| run: uv run dbt compile --target ${{ env.DBT_TARGET }} | |
| - name: Generate dbt documentation | |
| working-directory: warehouse | |
| run: uv run dbt docs generate --target ${{ env.DBT_TARGET }} --no-compile | |
| - name: Archive compilation artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dbt | |
| path: | | |
| warehouse/target/*.html | |
| warehouse/target/*.json | |
| warehouse/target/*.msgpack | |
| impacted_exposures: | |
| name: Impacted Exposures | |
| needs: [compile] | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| pull-requests: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download compilation artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dbt | |
| path: warehouse/target | |
| - name: Authenticate Google Service Account | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: 'true' | |
| project_id: ${{ env.PROJECT_ID }} | |
| workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.SERVICE_ACCOUNT }} | |
| - name: Setup GCloud utilities | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| working-directory: warehouse | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| enable-cache: true | |
| cache-python: true | |
| - name: Install the project | |
| working-directory: warehouse | |
| run: uv sync --locked --all-extras --dev | |
| - name: Install dbt dependencies | |
| working-directory: warehouse | |
| run: uv run dbt deps | |
| - name: List changed models | |
| uses: tj-actions/changed-files@v46 | |
| id: changed-models | |
| with: | |
| path: warehouse | |
| separator: "+ " | |
| files: models/**/*.sql | |
| - name: Find impacted exposures | |
| id: exposures | |
| if: ${{ steps.changed-models.outputs.any_changed == 'true' }} | |
| working-directory: warehouse | |
| run: | | |
| models="${{ steps.changed-models.outputs.all_changed_files }}" | |
| exposures=$(uv run dbt ls \ | |
| --select "${models}+" \ | |
| --resource-type exposure \ | |
| --output name \ | |
| --quiet) | |
| { | |
| echo "## Impacted Exposures" | |
| echo "" | |
| if [ -n "$exposures" ]; then | |
| echo "The following exposures are downstream of models changed in this PR:" | |
| echo "" | |
| echo "$exposures" | while IFS= read -r exp; do | |
| echo "- [\`${exp}\`](https://dbt-docs.dds.dot.ca.gov/#!/exposure/exposure.calitp_warehouse.${exp})" | |
| done | |
| else | |
| echo "_No exposures are impacted by the changes in this PR._" | |
| fi | |
| echo "" | |
| echo "### Changed models" | |
| echo "" | |
| echo "$models" | sed 's/+ /\n/g' | while IFS= read -r model; do | |
| echo "- \`${model}\`" | |
| done | |
| echo "" | |
| echo "> If any impacted exposures are unexpected, verify that your changes do not unintentionally affect downstream consumers." | |
| } > "$RUNNER_TEMP/exposure-comment.md" | |
| - name: Find existing exposures comment | |
| id: find-exposures-comment | |
| if: ${{ steps.changed-models.outputs.any_changed == 'true' }} | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: '## Impacted Exposures' | |
| - name: Post or update exposures comment | |
| if: ${{ steps.changed-models.outputs.any_changed == 'true' }} | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| comment-id: ${{ steps.find-exposures-comment.outputs.comment-id }} | |
| issue-number: ${{ github.event.number }} | |
| body-path: ${{ runner.temp }}/exposure-comment.md | |
| edit-mode: replace | |
| upload_latest_artifacts: | |
| name: Upload to latest dbt artifacts folder (Staging) | |
| needs: [compile] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| steps: | |
| - name: Download compilation artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dbt | |
| path: warehouse/target | |
| - name: Authenticate Google Service Account | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: 'true' | |
| project_id: ${{ env.PROJECT_ID }} | |
| workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.SERVICE_ACCOUNT }} | |
| - name: Setup GCloud utilities | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Upload catalog.json, manifest.json, and run_results.json | |
| uses: google-github-actions/upload-cloud-storage@v2 | |
| with: | |
| path: './warehouse/target/' | |
| glob: '{catalog.json,manifest.json,run_results.json}' | |
| parent: false | |
| process_gcloudignore: false | |
| destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/latest/" | |
| headers: |- | |
| content-type: application/json | |
| - name: Upload index.html | |
| uses: google-github-actions/upload-cloud-storage@v2 | |
| with: | |
| path: './warehouse/target/' | |
| glob: 'index.html' | |
| parent: false | |
| process_gcloudignore: false | |
| destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/latest/" | |
| headers: |- | |
| content-type: text/html; charset=utf-8 | |
| - name: Upload partial_parse.msgpack | |
| uses: google-github-actions/upload-cloud-storage@v2 | |
| with: | |
| path: './warehouse/target/' | |
| glob: 'partial_parse.msgpack' | |
| parent: false | |
| process_gcloudignore: false | |
| destination: "${{ env.DBT_ARTIFACTS_BUCKET }}/latest/" | |
| headers: |- | |
| content-type: application/vnd.msgpack |