Fix query type detection: scan comments with the server lexer rules

CHANGELOG.md

@@ -8,6 +8,7 @@
 - `None` nested inside an `Array` or `Tuple`, or inside a `Map` when `dict_parameter_format="map"`, now renders as the SQL `NULL` keyword instead of the `\N` sentinel used for top-level values. Top-level scalar `None` binds are unchanged. Closes [#879](https://github.com/ClickHouse/clickhouse-connect/issues/879).
 - Inserting empty bytes `b""` into a non-nullable `FixedString(N)` column now zero-pads to N bytes instead of raising `DataError`, matching the existing string and nullable-bytes write paths. Closes [#880](https://github.com/ClickHouse/clickhouse-connect/issues/880).
 - Per-query and client settings that are not present in `system.settings` for the current user (including custom settings declared `CHANGEABLE_IN_READONLY` on a role) are now forwarded to ClickHouse instead of raising `ProgrammingError: Setting ... is unknown or readonly`. The client cannot discover those settings without extra privileges, so the server is treated as authoritative. Setting `invalid_setting_action` to `drop` still drops them, so a single settings dict stays portable across server versions. Known readonly settings still honor `invalid_setting_action`, and reserved HTTP request parameter names such as `query`, `user`, `default_format`, and the `param_` bound-parameter namespace still raise a client-side `ProgrammingError` because they are not settings. Closes [#530](https://github.com/ClickHouse/clickhouse-connect/issues/530).
+- Query type detection now understands every comment and quoting form the server lexer accepts, so a comment can no longer change how a query is handled. `remove_sql_comments` only recognized `--` line comments, non-nested `/* */` block comments and `'`/`"` quoting, so a `LIMIT` inside a `//` or `# ` comment or after the inner `*/` of a nested block comment looked like a real `LIMIT` (the client side `query_limit` was silently dropped, and a commented out `LIMIT 0` routed the query to the columns-only metadata probe and returned no rows), while a `--` inside a backtick quoted identifier, a `$tag$` heredoc or a backslash escaped string truncated the query so the real trailing `LIMIT` was lost and the client appended a second one, which the server rejected with `Code: 62`. Comments are now removed with a single linear scan that follows the server lexer: `--`, `//`, `# ` and `#!` line comments, nested `/* */` block comments, `''`, `""` and `` `` `` quoting with backslash and doubled quote escapes, and `$tag$` heredocs. A query with an unterminated comment or quote is passed through unchanged so the server reports the syntax error. Closes [#925](https://github.com/ClickHouse/clickhouse-connect/issues/925).
 - The native streaming response buffer again detects mid-stream server exceptions proactively. Its in-band exception scan built the markers as `__exception__<tag>` and `<tag>__exception__`, but the server separates `__exception__` from the tag with a CRLF on both markers (`__exception__\r\n<tag>` ... `<tag>\r\n__exception__`), so the scan never matched and the exception block was only recovered by the last-chunk fallback in `NativeTransform.parse_response`. When the block spanned a transport-chunk boundary that fallback saw just a fragment and surfaced a truncated or garbled error instead of the real ClickHouse exception. Both the pure Python and compiled Cython buffers are corrected. Closes [#915](https://github.com/ClickHouse/clickhouse-connect/issues/915).
 
 ## 1.6.0, 2026-07-23

clickhouse_connect/driver/query.py

@@ -404,27 +404,119 @@ def close(self) -> None:
             self._block_gen = None
 
 
-comment_re = re.compile(r"(\".*?\"|\'.*?\')|(/\*.*?\*/|(--)[^\n]*$)", re.MULTILINE | re.DOTALL)
+# Everything that can start a comment or a quoted token in the server lexer. A plain
+# alternation of literals, so the search itself cannot backtrack.  A "$" is a word character
+# for the server lexer, so a heredoc tag only opens one at the start of a token.
+_comment_scan_re = re.compile(r"['\"`]|--|//|#[ !]|/\*|(?<![\w$])\$\w*\$")
+
+
+def _end_of_line_comment(sql: str, pos: int) -> int:
+    """Return the index of the newline that ends a "--", "//", "# " or "#!" comment."""
+    end = sql.find("\n", pos)
+    return len(sql) if end < 0 else end
+
+
+def _end_of_block_comment(sql: str, pos: int) -> int:
+    """Return the index after a "/* */" comment, or -1 if it is not closed.
+
+    Block comments nest, so an inner "/*" has to be matched by its own "*/".
+    """
+    depth = 0
+    while True:
+        opened = sql.find("/*", pos)
+        closed = sql.find("*/", pos)
+        if closed < 0:
+            return -1
+        if 0 <= opened < closed:
+            depth += 1
+            pos = opened + 2
+            continue
+        depth -= 1
+        pos = closed + 2
+        if depth == 0:
+            return pos
+
+
+def _end_of_quoted(sql: str, pos: int) -> int:
+    """Return the index after a quoted string or identifier, or -1 if it is not closed.
+
+    Accepts backslash escapes (\\X) and doubled quote escapes ('', "", ``).
+    """
+    quote = sql[pos]
+    pos += 1
+    end = len(sql)
+    while pos < end:
+        char = sql[pos]
+        if char == "\\":
+            pos += 2
+        elif char == quote:
+            if sql[pos + 1 : pos + 2] == quote:
+                pos += 2
+            else:
+                return pos + 1
+        else:
+            pos += 1
+    return -1
+
+
+def _end_of_heredoc(sql: str, pos: int, tag: str) -> int:
+    """Return the index after a "$tag$ ... $tag$" heredoc, or -1 if it is not closed."""
+    closed = sql.find(tag, pos + len(tag))
+    return -1 if closed < 0 else closed + len(tag)
 
 
 def remove_sql_comments(sql: str) -> str:
     """
     Remove SQL comments.  This is useful to determine the type of SQL query, such as SELECT or INSERT, but we
     don't fully trust it to correctly ignore weird quoted strings, and other edge cases, so we always pass the
     original SQL to ClickHouse (which uses a full-fledged AST/ token parser)
+
+    The query is scanned once, left to right, following the server lexer: "--", "//", "# " and "#!" line
+    comments, nested "/* */" block comments, '', "" and `` quoting, and "$tag$" heredocs.  A comment marker
+    inside a quoted token or a heredoc is not a comment, and a quote inside a comment does not start a quoted
+    token.  The scan jumps from one such token to the next, so it stays linear in the length of the query.  A
+    query with an unterminated comment or quote is rejected by the server, so the rest of it is kept as is,
+    while an unterminated heredoc tag is treated as a bare word, which is what the server lexer does.
     :param sql:  SQL query
     :return: SQL Query without SQL comments
     """
-
-    def replacer(match):
-        # if the 2nd group (capturing comments) is not None, it means we have captured a
-        # non-quoted, actual comment string, so return nothing to remove the comment
-        if match.group(2):
-            return ""
-        # Otherwise we've actually captured a quoted string, so return it
-        return match.group(1)
-
-    return comment_re.sub(replacer, sql)
+    kept: list[str] = []
+    pos = 0
+    while True:
+        match = _comment_scan_re.search(sql, pos)
+        if match is None:
+            kept.append(sql[pos:])
+            break
+        kept.append(sql[pos : match.start()])
+        token = match.group()
+        start = match.start()
+        if token in ("--", "//") or token[0] == "#":
+            # The newline itself is not part of the comment, it is copied on the next pass
+            pos = _end_of_line_comment(sql, start)
+        elif token == "/*":
+            end = _end_of_block_comment(sql, start)
+            if end < 0:
+                kept.append(sql[start:])
+                break
+            pos = end
+        elif token[0] == "$":
+            end = _end_of_heredoc(sql, start, token)
+            if end < 0:
+                # The server lexer falls back to a bare word when the closing tag is missing,
+                # so the scan continues after the text that looked like an opening tag
+                kept.append(token)
+                pos = match.end()
+                continue
+            kept.append(sql[start:end])
+            pos = end
+        else:
+            end = _end_of_quoted(sql, start)
+            if end < 0:
+                kept.append(sql[start:])
+                break
+            kept.append(sql[start:end])
+            pos = end
+    return "".join(kept)
 
 
 def to_arrow(content: bytes):

tests/integration_tests/test_client.py

@@ -280,6 +280,36 @@ def test_query_with_comment(param_client, call):
     assert len(result.result_set) > 0
 
 
+COMMENTED_LIMIT_QUERIES = [
+    # a LIMIT inside a comment is not a LIMIT, so the client side query_limit still applies
+    ("SELECT number FROM numbers(5) -- LIMIT 5", [(0,), (1,)]),
+    ("SELECT number FROM numbers(5) // LIMIT 5", [(0,), (1,)]),
+    ("SELECT number FROM numbers(5) # LIMIT 5", [(0,), (1,)]),
+    ("SELECT number FROM numbers(5) #!LIMIT 5", [(0,), (1,)]),
+    ("SELECT number FROM numbers(5) // LIMIT 0", [(0,), (1,)]),
+    ("SELECT number FROM numbers(5) /* LIMIT 5 */", [(0,), (1,)]),
+    ("SELECT number FROM numbers(5) /* a /* LIMIT 0 */ b */", [(0,), (1,)]),
+    ("SELECT number AS `a$b$c` FROM numbers(5) // LIMIT 5", [(0,), (1,)]),
+    # a comment marker inside a quoted identifier, string or heredoc is not a comment, so
+    # the real trailing LIMIT is honored and the client does not append a second one
+    ("SELECT number AS `a--b` FROM numbers(9) LIMIT 1", [(0,)]),
+    ("SELECT number, $$--$$ AS tag FROM numbers(9) LIMIT 1", [(0, "--")]),
+    ("SELECT number FROM numbers(9) WHERE toString(number) != 'x-- LIMIT 0' LIMIT 1", [(0,)]),
+    ("SELECT number FROM numbers(9) WHERE toString(number) != 'a\\'b-- LIMIT 0' LIMIT 1", [(0,)]),
+]
+
+
+@pytest.mark.parametrize(("query", "expected_rows"), COMMENTED_LIMIT_QUERIES)
+def test_query_limit_with_comments(param_client, call, query: str, expected_rows: list[tuple]):
+    old_limit = param_client.query_limit
+    param_client.query_limit = 2
+    try:
+        result = call(param_client.query, query)
+    finally:
+        param_client.query_limit = old_limit
+    assert result.result_rows == expected_rows
+
+
 def test_insert_csv_format(param_client, call, test_table_engine: str):
     call(param_client.command, "DROP TABLE IF EXISTS test_csv")
     call(

tests/unit_tests/test_driver/test_parser.py

@@ -150,3 +150,39 @@ def test_remove_comments_no_space_after_dashes():
     # `--` inside quoted strings is preserved
     assert remove_sql_comments("SELECT 'a--b'") == "SELECT 'a--b'"
     assert remove_sql_comments('SELECT "a--b"') == 'SELECT "a--b"'
+
+
+@pytest.mark.parametrize(
+    ("sql", "expected"),
+    [
+        # line comment forms the server lexer accepts
+        ("SELECT 13 // LIMIT 5", "SELECT 13 "),
+        ("SELECT 13 # LIMIT 5", "SELECT 13 "),
+        ("SELECT 13 #!LIMIT 5", "SELECT 13 "),
+        ("SELECT 13 // LIMIT 5\nLIMIT 3", "SELECT 13 \nLIMIT 3"),
+        ("SELECT 13 -- // # /*", "SELECT 13 "),
+        # a bare `#` is not a comment marker, it needs a following space or `!`
+        ("SELECT 13 #LIMIT 5", "SELECT 13 #LIMIT 5"),
+        ("SELECT 13 #\tLIMIT 5", "SELECT 13 #\tLIMIT 5"),
+        # block comments nest, so the inner `*/` does not end the outer comment
+        ("SELECT /* a /* b */ c */ 13", "SELECT  13"),
+        ("SELECT /* a /* b /* c */ d */ e */ 13", "SELECT  13"),
+        # a comment marker inside a quoted identifier, string or heredoc is not a comment
+        ("SELECT `a--b` FROM tbl", "SELECT `a--b` FROM tbl"),
+        ("SELECT `a``b--c` FROM tbl", "SELECT `a``b--c` FROM tbl"),
+        ("SELECT 'a''b--c'", "SELECT 'a''b--c'"),
+        ("SELECT 'a\\'b--c'", "SELECT 'a\\'b--c'"),
+        ("SELECT $$--$$", "SELECT $$--$$"),
+        ("SELECT $tag$ // $tag$", "SELECT $tag$ // $tag$"),
+        ("SELECT '--' // trailing", "SELECT '--' "),
+        # a `$` is a word character for the server lexer, so it only opens a heredoc at the
+        # start of a token, and an unclosed tag is just a word
+        ("SELECT 13 AS a$b$c -- LIMIT 0", "SELECT 13 AS a$b$c "),
+        ("SELECT 13 $notatag$ -- LIMIT 0", "SELECT 13 $notatag$ "),
+        # an unterminated comment or quote is left alone, the server reports it
+        ("SELECT 13 /* unclosed", "SELECT 13 /* unclosed"),
+        ("SELECT 'unclosed--", "SELECT 'unclosed--"),
+    ],
+)
+def test_remove_comments_lexer_forms(sql, expected):
+    assert remove_sql_comments(sql) == expected