Merge pull request #681 from PolicyEngine/fix/cicd-restructure

Restructure CI/CD pipeline and reorganize tests

Author: baogorek

.codecov.yml

@@ -0,0 +1,12 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true
+
+comment:
+  layout: "condensed_header, condensed_files"
+  behavior: default

.github/workflows/code_changes.yaml

@@ -29,7 +29,6 @@ on:
 jobs:
   pipeline:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
 
@@ -64,5 +63,15 @@ jobs:
               num_workers=int('${NUM_WORKERS}'),
               skip_national='${SKIP_NATIONAL}' == 'true',
           )
-          print(f'Pipeline spawned. Monitor on the Modal dashboard.')
+          print(f'Pipeline spawned.')
+          print(f'Function call ID: {fc.object_id}')
+
+          with open('$GITHUB_STEP_SUMMARY', 'a') as f:
+              f.write('## Pipeline Launched\n\n')
+              f.write('| Field | Value |\n')
+              f.write('|-------|-------|\n')
+              f.write(f'| GPU | \`${GPU}\` |\n')
+              f.write(f'| Epochs | \`${EPOCHS}\` / \`${NATIONAL_EPOCHS}\` |\n')
+              f.write(f'| Function call ID | \`{fc.object_id}\` |\n\n')
+              f.write('**[Monitor on Modal Dashboard](https://modal.com/apps)**\n')
           "

.github/workflows/pipeline.yaml

@@ -0,0 +1,151 @@
+name: PR checks
+
+on:
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: pr-checks-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  check-fork:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if PR is from fork
+        run: |
+          if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
+            echo "::error::PRs must be from branches in PolicyEngine/policyengine-us-data, not forks."
+            exit 1
+          fi
+
+  check-lock-freshness:
+    runs-on: ubuntu-latest
+    needs: check-fork
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - uses: astral-sh/setup-uv@v5
+      - name: Check lock file is up-to-date
+        run: |
+          uv lock --locked || {
+            echo "::error::uv.lock is outdated. Run 'uv lock' and commit."
+            exit 1
+          }
+
+  lint:
+    runs-on: ubuntu-latest
+    needs: check-fork
+    steps:
+      - uses: actions/checkout@v4
+      - run: pip install ruff>=0.9.0
+      - run: ruff format --check .
+
+  check-changelog:
+    runs-on: ubuntu-latest
+    needs: check-fork
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check for changelog fragment
+        run: |
+          FRAGMENTS=$(find changelog.d -type f ! -name '.gitkeep' | wc -l)
+          if [ "$FRAGMENTS" -eq 0 ]; then
+            echo "::error::No changelog fragment in changelog.d/"
+            exit 1
+          fi
+
+  unit-tests:
+    runs-on: ubuntu-latest
+    needs: [check-fork, lint]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - uses: astral-sh/setup-uv@v5
+      - run: uv sync --dev
+      - name: Run unit tests with coverage
+        env:
+          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+        run: >
+          uv run pytest tests/unit/
+          --cov=policyengine_us_data
+          --cov-report=xml
+          -v
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v4
+        with:
+          file: coverage.xml
+          flags: unit
+          fail_ci_if_error: false
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  smoke-test:
+    runs-on: ubuntu-latest
+    needs: [check-fork, lint]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - run: python -m pip install .
+      - run: python -c "import policyengine_us_data; print('OK')"
+      - run: python -c "from policyengine_core.data import Dataset; print('OK')"
+
+  docs-build:
+    runs-on: ubuntu-latest
+    needs: [check-fork]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+      - uses: astral-sh/setup-uv@v5
+      - run: uv sync --dev
+      - name: Test documentation builds
+        run: uv run make documentation
+
+  decide-test-scope:
+    runs-on: ubuntu-latest
+    needs: check-fork
+    outputs:
+      run_integration: ${{ steps.check.outputs.run_integration }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Check changed files for integration scope
+        id: check
+        run: |
+          CHANGED=$(git diff --name-only origin/main...HEAD)
+          if echo "$CHANGED" | grep -qE '^(policyengine_us_data/|modal_app/|tests/integration/)'; then
+            echo "run_integration=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_integration=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: [check-fork, lint, decide-test-scope]
+    if: needs.decide-test-scope.outputs.run_integration == 'true'
+    env:
+      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+      HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - run: pip install modal
+      - name: Build datasets and run integration tests on Modal
+        run: |
+          modal run modal_app/data_build.py \
+            --branch=${{ github.head_ref || github.ref_name }}