fix: protect SQLAlchemy reflection from String bytes format #1748
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Lint and Test' | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'VERSION' | |
| - 'LICENSE' | |
| - '**.md' | |
| - 'examples' | |
| - 'publish.yaml' | |
| - '.github/workflows/clickhouse_ci.yml' | |
| - '.github/workflows/on_push.yml' | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'VERSION' | |
| - 'LICENSE' | |
| - '**.md' | |
| - 'examples' | |
| - 'publish.yaml' | |
| - '.github/workflows/clickhouse_ci.yml' | |
| - '.github/workflows/on_push.yml' | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| name: Ruff | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Install Ruff | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install "ruff==0.15.8" | |
| - name: Ruff format check | |
| run: ruff format --check clickhouse_connect tests examples setup.py | |
| - name: Ruff lint check | |
| run: ruff check clickhouse_connect tests examples setup.py | |
| typecheck: | |
| runs-on: ubuntu-latest | |
| name: Mypy | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r tests/test_requirements.txt | |
| - name: Mypy check | |
| run: mypy | |
| public-api-types: | |
| runs-on: ubuntu-latest | |
| name: Public API Type Surface | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install into a clean consumer environment | |
| run: | | |
| python -m venv "$RUNNER_TEMP/consumer" | |
| pip install build | |
| CLICKHOUSE_CONNECT_SKIP_CYTHON=1 python -m build --sdist --outdir "$RUNNER_TEMP/dist" | |
| CLICKHOUSE_CONNECT_SKIP_CYTHON=1 "$RUNNER_TEMP/consumer/bin/pip" install "$RUNNER_TEMP"/dist/*.tar.gz | |
| # Stub packages are intentionally unpinned. If the ratchet count shifts without a repo change, pin them here. | |
| "$RUNNER_TEMP/consumer/bin/pip" install numpy pandas pandas-stubs pyarrow polars "sqlalchemy>=2.0,<3.0" | |
| "$RUNNER_TEMP/consumer/bin/pip" install "mypy==2.1.0" "pyright==1.1.408" | |
| - name: Confirm py.typed shipped with the install | |
| run: test -f "$RUNNER_TEMP"/consumer/lib/python*/site-packages/clickhouse_connect/py.typed | |
| - name: Consumer smoke test (mypy --strict against installed wheel) | |
| run: | | |
| cp tests/type_check/consumer_smoke.py "$RUNNER_TEMP" | |
| cd "$RUNNER_TEMP" | |
| "$RUNNER_TEMP/consumer/bin/mypy" --strict --follow-imports=silent consumer_smoke.py | |
| - name: Public type completeness ratchet | |
| # SQLAlchemy must be installed while verifytypes scans the public cc_sqlalchemy annotations. | |
| # The pre-feature baseline is 1120 with that optional surface resolved. | |
| run: python scripts/check_public_types.py clickhouse_connect --max-untyped 1120 --python "$RUNNER_TEMP/consumer/bin/python" | |
| - name: SQLAlchemy Select typing smoke test | |
| run: | | |
| cp tests/type_check/sqlalchemy_select_smoke.py "$RUNNER_TEMP" | |
| cd "$RUNNER_TEMP" | |
| "$RUNNER_TEMP/consumer/bin/mypy" --strict --follow-imports=silent sqlalchemy_select_smoke.py | |
| bare-import-test: | |
| runs-on: ubuntu-latest | |
| needs: [lint, typecheck, public-api-types] | |
| name: Bare Install Test (no extras) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Start ClickHouse (latest) in Docker | |
| env: | |
| CLICKHOUSE_CONNECT_TEST_CH_VERSION: latest | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml up -d --wait | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install only core dependencies (no extras) | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e . --no-deps | |
| pip install certifi urllib3 backports.zstd lz4 | |
| - name: Bare import test and basic query | |
| run: python tests/test_bare_install.py | |
| - name: Stop ClickHouse | |
| if: ${{ always() && hashFiles('docker-compose.yml') != '' }} | |
| env: | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml down --volumes --remove-orphans | |
| tests: | |
| runs-on: ubuntu-latest | |
| needs: bare-import-test | |
| strategy: | |
| matrix: | |
| python-version: | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| - '3.13' | |
| - '3.14' | |
| clickhouse-version: | |
| - '25.8' # LTS | |
| - '26.3' # LTS | |
| - '26.5' # Stable | |
| - '26.6' # Stable | |
| - '26.7' # Stable | |
| use-c: | |
| - '1' | |
| include: | |
| - python-version: '3.10' | |
| clickhouse-version: '26.3' | |
| use-c: '0' | |
| - python-version: '3.14' | |
| clickhouse-version: '26.3' | |
| use-c: '0' | |
| # Free-threaded build (experimental). PYTHON_GIL=0 keeps the GIL off when | |
| # dependencies that have not declared free-threading compatibility are imported. | |
| - python-version: '3.14t' | |
| clickhouse-version: '26.3' | |
| use-c: '1' | |
| python-gil: '0' | |
| # The free-threaded job is non-blocking while support is experimental | |
| continue-on-error: ${{ matrix.python-version == '3.14t' }} | |
| name: Local Tests Py=${{ matrix.python-version }} CH=${{ matrix.clickhouse-version }} C=${{ matrix.use-c }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Start ClickHouse (version - ${{ matrix.clickhouse-version }}) in Docker | |
| env: | |
| CLICKHOUSE_CONNECT_TEST_CH_VERSION: ${{ matrix.clickhouse-version }} | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml up -d --wait | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install Test Dependencies | |
| # polars ships no free-threaded wheels and its sdist needs a long Rust nightly | |
| # build, so it is excluded on 3.14t and its tests skip | |
| run: | | |
| if [ "${{ matrix.python-version }}" = "3.14t" ]; then | |
| grep -v '^polars' tests/test_requirements.txt > /tmp/test_requirements.txt | |
| pip install -r /tmp/test_requirements.txt | |
| else | |
| pip install -r tests/test_requirements.txt | |
| fi | |
| - name: Build cython extensions | |
| run: python setup.py build_ext --inplace | |
| - name: "Add distribution info" # This lets SQLAlchemy find entry points | |
| run: pip install -e . --no-deps | |
| - name: Add ClickHouse TLS instance to /etc/hosts | |
| run: | | |
| sudo echo "127.0.0.1 server1.clickhouse.test" | sudo tee -a /etc/hosts | |
| - name: Run tests | |
| env: | |
| CLICKHOUSE_CONNECT_USE_C: ${{ matrix.use-c }} | |
| CLICKHOUSE_CONNECT_TEST_TLS: 1 | |
| CLICKHOUSE_CONNECT_TEST_DOCKER: 'False' | |
| CLICKHOUSE_CONNECT_TEST_FUZZ: 50 | |
| SQLALCHEMY_SILENCE_UBER_WARNING: 1 | |
| # Empty for non-free-threaded entries, CPython ignores an empty value | |
| PYTHON_GIL: ${{ matrix.python-gil }} | |
| run: pytest -n 4 tests | |
| - name: Stop ClickHouse | |
| if: ${{ always() && hashFiles('docker-compose.yml') != '' }} | |
| env: | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml down --volumes --remove-orphans | |
| pandas-3x-compat-test: | |
| runs-on: ubuntu-latest | |
| needs: bare-import-test | |
| name: Pandas 3.x Compatibility Tests | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Start ClickHouse (version - latest) in Docker | |
| env: | |
| CLICKHOUSE_CONNECT_TEST_CH_VERSION: latest | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml up -d --wait | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install Test Dependencies | |
| run: pip install -r tests/test_requirements.txt | |
| - name: Install pandas 3.x | |
| run: pip install "pandas>=3,<4" # Override pandas version | |
| - name: Build cython extensions | |
| run: python setup.py build_ext --inplace | |
| - name: "Add distribution info" # This lets SQLAlchemy find entry points | |
| run: pip install -e . --no-deps | |
| - name: Add ClickHouse TLS instance to /etc/hosts | |
| run: | | |
| sudo echo "127.0.0.1 server1.clickhouse.test" | sudo tee -a /etc/hosts | |
| - name: Run tests | |
| env: | |
| CLICKHOUSE_CONNECT_TEST_TLS: 1 | |
| CLICKHOUSE_CONNECT_TEST_DOCKER: 'False' | |
| SQLALCHEMY_SILENCE_UBER_WARNING: 1 | |
| run: pytest -n 4 tests/unit_tests/test_pandas.py tests/integration_tests/test_pandas.py tests/integration_tests/test_async_features.py -k "pandas or arrow" | |
| - name: Stop ClickHouse | |
| if: ${{ always() && hashFiles('docker-compose.yml') != '' }} | |
| env: | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml down --volumes --remove-orphans | |
| sqlalchemy-1x-compat-test: | |
| runs-on: ubuntu-latest | |
| needs: bare-import-test | |
| name: SQLAlchemy 1.x Compatibility Tests | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Start ClickHouse (version - latest) in Docker | |
| env: | |
| CLICKHOUSE_CONNECT_TEST_CH_VERSION: latest | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml up -d --wait | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.11' | |
| - name: Install pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install Test Dependencies | |
| run: pip install -r tests/test_requirements.txt | |
| - name: Install sqlalchemy 1.x | |
| run: pip install "sqlalchemy>=1.4.40,<2.0" # Override sqa version | |
| - name: Build cython extensions | |
| run: python setup.py build_ext --inplace | |
| - name: "Add distribution info" # This lets SQLAlchemy find entry points | |
| run: pip install -e . --no-deps | |
| - name: Add ClickHouse TLS instance to /etc/hosts | |
| run: | | |
| sudo echo "127.0.0.1 server1.clickhouse.test" | sudo tee -a /etc/hosts | |
| - name: Run tests | |
| env: | |
| CLICKHOUSE_CONNECT_TEST_TLS: 1 | |
| CLICKHOUSE_CONNECT_TEST_DOCKER: 'False' | |
| SQLALCHEMY_SILENCE_UBER_WARNING: 1 | |
| run: pytest -n 4 tests/integration_tests/test_sqlalchemy tests/unit_tests/test_sqlalchemy/test_ch_select.py tests/unit_tests/test_sqlalchemy/test_materialized_cte.py | |
| - name: Stop ClickHouse | |
| if: ${{ always() && hashFiles('docker-compose.yml') != '' }} | |
| env: | |
| COMPOSE_PROJECT_NAME: clickhouse-connect-ci | |
| run: docker compose -f docker-compose.yml down --volumes --remove-orphans | |
| check-secret: | |
| runs-on: ubuntu-latest | |
| needs: [bare-import-test, tests, pandas-3x-compat-test, sqlalchemy-1x-compat-test] | |
| outputs: | |
| has_secrets: ${{ steps.has_secrets.outputs.HAS_SECRETS }} | |
| steps: | |
| - id: has_secrets | |
| env: | |
| CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST }} | |
| if: "${{ env.CLOUD_HOST != '' }}" | |
| run: echo "HAS_SECRETS=true" >> $GITHUB_OUTPUT | |
| cloud-tests: | |
| runs-on: ubuntu-latest | |
| name: Cloud Tests Py=3.12 | |
| needs: check-secret | |
| if: needs.check-secret.outputs.has_secrets == 'true' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r tests/test_requirements.txt | |
| - name: Build cython extensions | |
| run: python setup.py build_ext --inplace | |
| - name: "Add distribution info" # This lets SQLAlchemy find entry points | |
| run: pip install -e . --no-deps | |
| - name: Run tests | |
| env: | |
| CLICKHOUSE_CONNECT_TEST_FUZZ: 10 | |
| CLICKHOUSE_CONNECT_TEST_CLOUD: 'True' | |
| CLICKHOUSE_CONNECT_TEST_PORT: 8443 | |
| CLICKHOUSE_CONNECT_TEST_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT_PROD }} | |
| CLICKHOUSE_CONNECT_TEST_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT_PROD }} | |
| CLICKHOUSE_CONNECT_TEST_JWT_SECRET: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_JWT_DESERT_VM_43 }} | |
| SQLALCHEMY_SILENCE_UBER_WARNING: 1 | |
| run: pytest -n 4 tests/integration_tests |