fix[ci]: modernize CI and petl dependencies #362
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a variety of Python versions and Operating Systems | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: Test Changes | |
| on: | |
| push: | |
| branches: ["master"] | |
| merge_group: | |
| branches: ["master"] | |
| pull_request: | |
| branches: ["master"] | |
| types: [opened, reopened, synchronize, ready_for_review] | |
| workflow_dispatch: | |
| inputs: | |
| branch_commit_or_ref: | |
| description: "Run this workflow in what branch/commit?" | |
| required: true | |
| type: string | |
| default: master | |
| jobs: | |
| run-guard: | |
| # it succeeds if any of the following conditions are met: | |
| # - when the PR is not a draft and is not labeled 'prevent-ci' | |
| # - when the PR is labeled 'force-ci' | |
| runs-on: ubuntu-latest | |
| if: | | |
| ( (!github.event.pull_request.draft) && | |
| (github.event.action != 'labeled') && | |
| (!contains( github.event.pull_request.labels.*.name, 'prevent-ci')) ) | |
| || ((github.event.action != 'labeled') && contains( github.event.pull_request.labels.*.name, 'force-ci')) | |
| || (github.event.label.name == 'force-ci') | |
| steps: | |
| - name: Checking if CI shoud run for ${{ github.ref_name }}... | |
| run: echo "Resuming CI. Continuing next jobs..." | |
| test-source-code: | |
| needs: run-guard | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| #@ https://github.com/actions/runner-images#available-images | |
| os: | |
| ["ubuntu-latest", "windows-latest", "macos-latest"] | |
| #@ https://github.com/marketplace/actions/setup-python | |
| #@ https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#python | |
| python: | |
| ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] | |
| # TODO: Also test with PyPy and GraalPy @https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#graalpy | |
| include: | |
| # Default configuration for all jobs | |
| - mode: "full" | |
| - scope: "full" | |
| - install: "no" | |
| - primary: "3.8" | |
| # Conditional configuration overrides | |
| - python: "3.6" | |
| install: "yes" | |
| - python: "3.7" | |
| install: "yes" | |
| - os: "windows-latest" | |
| mode: "basic" | |
| scope: "basic" | |
| # Handle random test failures | |
| - os: "ubuntu-latest" | |
| python: "3.9" | |
| mode: "basic" | |
| scope: "basic" | |
| # Additional Jobs not included in the matrix above | |
| - python: "2.7" | |
| os: "ubuntu-latest" | |
| mode: "basic" | |
| scope: "full" | |
| install: "yes" | |
| exclude: | |
| # macos-latest == macos-15-arm64 | |
| # macos-latest-large == macos-15-intel | |
| - os: "macos-latest" | |
| python: "3.6" | |
| - os: "macos-latest" | |
| python: "3.7" | |
| # python 3.7 too slow on ubuntu-latest | |
| - os: "ubuntu-latest" | |
| python: "3.7" | |
| runs-on: "${{ matrix.os }}" | |
| steps: | |
| - name: Checkout source code at ${{ inputs.branch_commit_or_ref || github.ref }} | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ inputs.branch_commit_or_ref || github.ref }} | |
| - name: Install linux tools | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| echo "::group::Install linux tools" | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends python3-h5py | |
| echo "::endgroup::" | |
| - name: Install additional linux tools | |
| if: matrix.os == 'ubuntu-latest' && matrix.scope == 'full' | |
| run: | | |
| echo "::group::Install additional linux tools" | |
| sudo apt-get install -y --no-install-recommends default-libmysqlclient-dev | |
| echo "::endgroup::" | |
| - name: Set up Python ${{ matrix.python }} with install = ${{ matrix.install }} | |
| if: matrix.install == 'no' | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Set up Python ${{ matrix.python }} discontinued on ${{ matrix.os }} | |
| if: matrix.install == 'yes' | |
| uses: MatteoH2O1999/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| cache: pip | |
| - name: Upgrade pip | |
| run: | | |
| echo "::group::Upgrading pip" | |
| python -m pip install --upgrade pip | |
| echo "::endgroup::" | |
| - name: Report Workflow Information | |
| id: workflow_report | |
| if: matrix.python >= '3' | |
| uses: ./.github/actions/workflow-info | |
| with: | |
| title: "${{ github.ref_name }} on ${{ matrix.os }} with python ${{ matrix.python }}" | |
| parameters: "${{ toJson(inputs) }}" | |
| content: | | |
| - Ref: ${{ github.head_ref }} | |
| - matrix.python: ${{ matrix.python }} | |
| - matrix.os: ${{ matrix.os }} | |
| - matrix.mode: ${{ matrix.mode }} | |
| - matrix.scope: ${{ matrix.scope }} | |
| - matrix.install: ${{ matrix.install }} | |
| - name: Install test dependencies | |
| run: | | |
| echo "::group::Install test dependencies" | |
| python -m pip install --prefer-binary -r requirements-tests.txt | |
| echo "::endgroup::" | |
| - name: Setup environment variables for remote filesystem testing with primary = ${{ matrix.primary }} | |
| if: matrix.os == 'ubuntu-latest' && matrix.python == matrix.primary | |
| run: | | |
| echo "Setup SMB environment variable to trigger testing in Petl" | |
| echo 'PETL_TEST_SMB=smb://WORKGROUP;petl:test@localhost/public/' >> $GITHUB_ENV | |
| echo "Setup SFTP environment variable to trigger testing in Petl" | |
| echo 'PETL_TEST_SFTP=sftp://petl:test@localhost:2244/public/' >> $GITHUB_ENV | |
| echo "::group::Install remote test dependencies" | |
| python -m pip install --prefer-binary -r requirements-remote.txt | |
| echo "::endgroup::" | |
| - name: Install containers for remote filesystem testing with install = ${{ matrix.install }} | |
| if: matrix.os == 'ubuntu-latest' && matrix.install == 'no' | |
| run: | | |
| echo "::group::Setup docker for SMB at: ${{ env.PETL_TEST_SMB }}$" | |
| docker run -it --name samba -p 139:139 -p 445:445 -d "dperson/samba" -p -u "petl;test" -s "public;/public-dir;yes;no;yes;all" | |
| echo "::endgroup::" | |
| echo "::group::Setup docker for SFTP at: ${{ env.PETL_TEST_SFTP }}$" | |
| docker run -it --name sftp -p 2244:22 -d atmoz/sftp petl:test:::public | |
| echo "::endgroup::" | |
| - name: Install containers for remote database testing with scope = ${{ matrix.scope }} | |
| if: matrix.os == 'ubuntu-latest' && matrix.scope == 'full' | |
| run: | | |
| echo "::group::Setup docker for MySQL" | |
| docker run -it --name mysql -p 3306:3306 -p 33060:33060 -e MYSQL_ROOT_PASSWORD=pass0 -e MYSQL_DATABASE=petl -e MYSQL_USER=petl -e MYSQL_PASSWORD=test -d mysql:latest | |
| echo "::endgroup::" | |
| echo "::group::Setup docker for Postgres" | |
| docker run -it --name postgres -p 5432:5432 -e POSTGRES_DB=petl -e POSTGRES_USER=petl -e POSTGRES_PASSWORD=test -d postgres:latest | |
| echo "::endgroup::" | |
| echo "::group::Install database test dependencies" | |
| python -m pip install --prefer-binary -r requirements-database.txt | |
| echo "::endgroup::" | |
| - name: Setup petl package with mode = ${{ matrix.mode }} | |
| run: | | |
| python -m pip install --upgrade build setuptools wheel | |
| python -m build | |
| python -m pip install build --user | |
| - name: Install extra packages dependencies for scope = ${{ matrix.scope }} | |
| if: matrix.scope == 'full' | |
| run: | | |
| echo "::group::Install extra packages dependencies" | |
| python -m pip install --prefer-binary -r requirements-formats.txt | |
| echo "::endgroup::" | |
| - name: Install optional test dependencies with install = ${{ matrix.install }} | |
| if: matrix.os == 'ubuntu-latest' && matrix.install == 'no' | |
| env: | |
| DISABLE_BLOSC_AVX2: 1 | |
| run: | | |
| echo "::group::Install tricky test dependencies" | |
| # pip install --prefer-binary 'Cython>=0.29.21,<3.0.0' bcolz | |
| if ! pip install --prefer-binary -r requirements-optional.txt ; then | |
| echo "::warning file=requirements-optional.txt,line=23,col=1,endColumn=5,title=Optional package failure::Dismissed failure installing some optional package. Resuming tests..." | |
| else | |
| echo "::notice file=requirements-optional.txt,line=23,col=1,endColumn=5,title=Optional package failure::Successfully installed all optional packages." | |
| fi | |
| echo "::endgroup::" | |
| - name: List Installed Packages for Throubleshooting | |
| shell: bash | |
| run: | | |
| echo "::group::List Installed Packages for Throubleshooting" | |
| printf '\n## Installed Packages for python ${{ matrix.python }}\n\n' >> "${GITHUB_STEP_SUMMARY}"; | |
| echo "- Python Version: $(python --version)" >> "${GITHUB_STEP_SUMMARY}"; | |
| echo "- Pytest Version: $(pytest --version)" >> "${GITHUB_STEP_SUMMARY}"; | |
| echo "- Pip Version: $(pip --version | head -n 1 | cut -d' ' -f2)" >> "${GITHUB_STEP_SUMMARY}"; | |
| printf '```text' >> "${GITHUB_STEP_SUMMARY}"; | |
| python -m pip list --format freeze >> "${GITHUB_STEP_SUMMARY}"; | |
| printf '```' >> "${GITHUB_STEP_SUMMARY}"; | |
| echo "::endgroup::" | |
| - name: Test python source code for mode = basic | |
| if: matrix.mode == 'basic' | |
| env: | |
| PYTHONTRACEMALLOC: 1 | |
| run: | | |
| echo "::group::Perform basic test execution with coverage" | |
| pytest --cov=petl petl | |
| echo "::endgroup::" | |
| - name: Test including documentation inside source code for mode = full | |
| if: matrix.mode == 'full' | |
| env: | |
| PYTHONTRACEMALLOC: 1 | |
| run: | | |
| echo "::group::Perform doctest-modules execution with coverage" | |
| pytest --doctest-modules --cov=petl --durations=10 petl | |
| echo "::endgroup::" | |
| - name: Coveralls with os = ${{ matrix.os }}, primary = ${{ matrix.primary }} | |
| if: matrix.os == 'ubuntu-latest' && matrix.python == matrix.primary | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| python -m pip install --upgrade coveralls | |
| coveralls --service=github | |
| - name: Print source code coverage | |
| run: | | |
| printf '\n## Coverage for python ${{ matrix.python }} with mode ${{ matrix.mode }}\n\n' >> "${GITHUB_STEP_SUMMARY}"; | |
| printf '```text' >> "${GITHUB_STEP_SUMMARY}"; | |
| coverage report -m >> "${GITHUB_STEP_SUMMARY}"; | |
| printf '```' >> "${GITHUB_STEP_SUMMARY}"; | |
| test-documentation: | |
| needs: run-guard | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python: [3.8] | |
| steps: | |
| - name: Checkout source code at ${{ inputs.branch_commit_or_ref || github.ref }} | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ inputs.branch_commit_or_ref || github.ref }} | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install doc generation dependencies | |
| run: | | |
| python -m pip install --prefer-binary -r requirements-docs.txt | |
| - name: Setup petl package | |
| run: | | |
| python -m pip install --upgrade build setuptools wheel | |
| python -m build | |
| - name: Test docs generation | |
| run: | | |
| cd docs | |
| sphinx-build -W -b singlehtml -d ../build/doctrees . ../build/singlehtml |