Live E2E tests #73
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Live E2E tests against the production / staging modelseed.org stack. | |
| # | |
| # - Smoke layer runs on every push to main and on a 6-hour schedule. | |
| # - Functional layer runs nightly (requires the MODELSEED_TEST_TOKEN secret). | |
| # - Biological layer runs weekly on Sunday (requires token + ~75 min). | |
| # - UI (Playwright) runs nightly when MODELSEED_TEST_UI_ENABLED is set. | |
| # | |
| # All jobs upload pytest-html reports as artifacts so failures are | |
| # diagnosable from the Actions tab without re-running locally. | |
| # | |
| # Required secrets (configure in GitHub Settings → Secrets and variables → Actions): | |
| # MODELSEED_TEST_TOKEN — PATRIC token for the test bot user | |
| # MODELSEED_TEST_RAST_TOKEN — (optional) separate RAST token if different from above | |
| # | |
| # Tokens are scoped to the live-tests job only and never echoed in logs. | |
| # See docs/E2E_TEST_PLAN.md for the full design. | |
| name: Live E2E tests | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| # Only run on pushes that could affect API behavior. | |
| - 'src/**' | |
| - 'tests/live/**' | |
| - 'pyproject.toml' | |
| - '.github/workflows/live-tests.yml' | |
| schedule: | |
| # Smoke every 6 hours, functional + UI nightly at 04:00 UTC, | |
| # biological weekly on Sundays at 06:00 UTC. We use one cron and | |
| # let the job conditions decide which layer to run. | |
| - cron: '0 */6 * * *' # smoke | |
| - cron: '0 4 * * *' # functional + UI nightly | |
| - cron: '0 6 * * 0' # biological weekly | |
| workflow_dispatch: | |
| # Allow manual run with optional layer selection. | |
| inputs: | |
| layer: | |
| description: 'Which layer to run' | |
| type: choice | |
| options: [smoke, functional, biological, ui, all] | |
| default: smoke | |
| target_env: | |
| description: 'Which environment to test' | |
| type: choice | |
| options: [production, staging] | |
| default: production | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| PYTHON_VERSION: '3.11' | |
| jobs: | |
| smoke: | |
| name: Smoke (no auth required) | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.event_name == 'push' || | |
| (github.event_name == 'workflow_dispatch' && (inputs.layer == 'smoke' || inputs.layer == 'all')) || | |
| github.event.schedule == '0 */6 * * *' || | |
| github.event.schedule == '0 4 * * *' | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: pip | |
| - name: Install | |
| run: pip install -e ".[dev]" | |
| - name: Run smoke layer | |
| env: | |
| MODELSEED_TEST_ENV: ${{ inputs.target_env || 'production' }} | |
| run: | | |
| mkdir -p tests/live/reports | |
| pytest tests/live/smoke/ \ | |
| --html=tests/live/reports/smoke.html --self-contained-html \ | |
| --json-report --json-report-file=tests/live/reports/smoke.json | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: smoke-report-${{ github.run_id }} | |
| path: tests/live/reports/ | |
| functional: | |
| name: Functional (requires PATRIC token) | |
| runs-on: ubuntu-latest | |
| needs: smoke | |
| if: | | |
| always() && needs.smoke.result != 'failure' && ( | |
| (github.event_name == 'workflow_dispatch' && (inputs.layer == 'functional' || inputs.layer == 'all')) || | |
| github.event.schedule == '0 4 * * *' || | |
| github.event.schedule == '0 6 * * 0' | |
| ) | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: pip | |
| - name: Install | |
| run: pip install -e ".[dev]" | |
| - name: Run functional layer | |
| env: | |
| MODELSEED_TEST_ENV: ${{ inputs.target_env || 'production' }} | |
| MODELSEED_TEST_TOKEN: ${{ secrets.MODELSEED_TEST_TOKEN }} | |
| run: | | |
| if [ -z "$MODELSEED_TEST_TOKEN" ]; then | |
| echo "::warning::MODELSEED_TEST_TOKEN secret not set — functional tests will skip auth-required cases" | |
| fi | |
| mkdir -p tests/live/reports | |
| pytest tests/live/functional/ \ | |
| --html=tests/live/reports/functional.html --self-contained-html \ | |
| --json-report --json-report-file=tests/live/reports/functional.json | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: functional-report-${{ github.run_id }} | |
| path: tests/live/reports/ | |
| ui: | |
| name: UI (Playwright) | |
| runs-on: ubuntu-latest | |
| needs: smoke | |
| if: | | |
| always() && needs.smoke.result != 'failure' && ( | |
| (github.event_name == 'workflow_dispatch' && (inputs.layer == 'ui' || inputs.layer == 'all')) || | |
| github.event.schedule == '0 4 * * *' | |
| ) | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: pip | |
| - name: Install + Playwright browsers | |
| run: | | |
| pip install -e ".[dev,ui-tests]" | |
| playwright install --with-deps chromium | |
| - name: Run UI layer | |
| env: | |
| MODELSEED_TEST_ENV: ${{ inputs.target_env || 'production' }} | |
| MODELSEED_TEST_TOKEN: ${{ secrets.MODELSEED_TEST_TOKEN }} | |
| MODELSEED_TEST_UI_ENABLED: '1' | |
| MODELSEED_TEST_UI_HEADLESS: '1' | |
| run: | | |
| mkdir -p tests/live/reports | |
| pytest tests/live/ui/ \ | |
| --html=tests/live/reports/ui.html --self-contained-html \ | |
| --json-report --json-report-file=tests/live/reports/ui.json | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ui-report-${{ github.run_id }} | |
| path: tests/live/reports/ | |
| biological: | |
| name: Biological (weekly, slow) | |
| runs-on: ubuntu-latest | |
| needs: [smoke, functional] | |
| if: | | |
| always() && needs.smoke.result != 'failure' && needs.functional.result != 'failure' && ( | |
| (github.event_name == 'workflow_dispatch' && (inputs.layer == 'biological' || inputs.layer == 'all')) || | |
| github.event.schedule == '0 6 * * 0' | |
| ) | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: pip | |
| - name: Install git-only modeling deps + dev extras | |
| run: | | |
| # kbutillib and cobrakbase aren't on PyPI; install from git first so | |
| # the [modeling] extra can resolve. | |
| pip install \ | |
| "git+https://github.com/cshenry/KBUtilLib.git@main" \ | |
| "git+https://github.com/Fxe/cobrakbase.git@master" | |
| pip install -e ".[dev,modeling]" | |
| - name: Run biological layer | |
| env: | |
| MODELSEED_TEST_ENV: ${{ inputs.target_env || 'production' }} | |
| MODELSEED_TEST_TOKEN: ${{ secrets.MODELSEED_TEST_TOKEN }} | |
| run: | | |
| if [ -z "$MODELSEED_TEST_TOKEN" ]; then | |
| echo "::error::MODELSEED_TEST_TOKEN secret required for biological layer" | |
| exit 1 | |
| fi | |
| mkdir -p tests/live/reports | |
| pytest tests/live/biological/ -n 4 \ | |
| --html=tests/live/reports/biological.html --self-contained-html \ | |
| --json-report --json-report-file=tests/live/reports/biological.json | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: biological-report-${{ github.run_id }} | |
| path: tests/live/reports/ |