Merge pull request #14 from Rappsilber-Laboratory/development

higher level columns pass-down

Author: z3rone

.github/workflows/python-package.yml .github/workflows/python-test.yml.github/workflows/python-package.yml renamed to .github/workflows/python-test.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: Python package
+name: Python test
 
 on:
   push:
@@ -15,18 +15,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         python -m pip install pytest
-        python -m pip install .
+        if [ "${{ endsWith(matrix.python-version, 't') }}" == "true" ]; then
+          python -m pip install ".[nogil]"
+        else
+          python -m pip install .
+        fi
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Test with pytest
       run: |

.python-version

@@ -1 +1 @@
-3.13
+3.14t

Pipfile

@@ -4,15 +4,21 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pandas = "*"
+pandas = ">=3.0.0"
 polars = "*"
-xifdr = {file = ".", editable = true}
+numpy = ">=2.1.0"
+scipy = ">=1.15.0"
+psutil = "*"
+maturin = "*"
+setuptools-rust = "*"
+kneed = "*"
 
 [dev-packages]
 sphinx = "*"
 sphinx-autodoc-typehints = "*"
 sphinx-rtd-theme = "*"
 pytest = "*"
+setuptools-scm = "*"
 
-[requires]
-python_version = "3.13"
+[pipenv]
+allow_prereleases = true

Pipfile.lock

@@ -12,6 +12,12 @@ Use `pip` to install xiFDR from PyPi:
 
     $ pip install xifdr
 
+If you use a free threaded python version install with the nogil option:
+
+.. code-block:: shell
+
+    $ pip install 'xifdr[nogil]'
+
 Input format
 ------------
 

docs/basic_usage.rst

@@ -12,15 +12,34 @@ authors = [
 license = { text = "AGPL-3.0-or-later" }
 readme = "README.rst"
 requires-python = ">=3.9"
+
+# Core dependencies (Standard behavior)
 dependencies = [
-    "numpy",
-    "pandas",
-    "polars",
-    "scipy",
+    "numpy>=2.1.0",
+    "pandas>=3.0.0",
+    "scipy>=1.15.0",
     "psutil",
     "pyarrow",
     "setuptools_scm",
     "kneed",
+    "polars",
+]
+
+[project.optional-dependencies]
+# Usage: pip install "xifdr[nogil]"
+nogil = [
+    # Free-threaded Polars Bridge for 3.13+
+    "polars @ https://github.com/Rappsilber-Laboratory/polars-free-threaded/releases/download/v1.36.0-beta.2-ft/polars-1.36.0b2-py3-none-any.whl ; python_version >= '3.13'",
+
+    # Platform-specific Polars Runtime for 3.13
+    "polars-runtime-32 @ https://github.com/Rappsilber-Laboratory/polars-free-threaded/releases/download/v1.36.0-beta.2-ft/polars_runtime_32-1.36.0b2-cp313-cp313t-macosx_11_0_arm64.whl ; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version == '3.13'",
+    "polars-runtime-32 @ https://github.com/Rappsilber-Laboratory/polars-free-threaded/releases/download/v1.36.0-beta.2-ft/polars_runtime_32-1.36.0b2-cp313-cp313t-manylinux_2_39_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.13'",
+    "polars-runtime-32 @ https://github.com/Rappsilber-Laboratory/polars-free-threaded/releases/download/v1.36.0-beta.2-ft/polars_runtime_32-1.36.0b2-cp313-cp313t-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.13'",
+
+    # Platform-specific Polars Runtime for 3.14
+    "polars-runtime-32 @ https://github.com/Rappsilber-Laboratory/polars-free-threaded/releases/download/v1.36.0-beta.2-ft/polars_runtime_32-1.36.0b2-cp314-cp314t-macosx_11_0_arm64.whl ; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version == '3.14'",
+    "polars-runtime-32 @ https://github.com/Rappsilber-Laboratory/polars-free-threaded/releases/download/v1.36.0-beta.2-ft/polars_runtime_32-1.36.0b2-cp314-cp314t-manylinux_2_39_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.14'",
+    "polars-runtime-32 @ https://github.com/Rappsilber-Laboratory/polars-free-threaded/releases/download/v1.36.0-beta.2-ft/polars_runtime_32-1.36.0b2-cp314-cp314t-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.14'",
 ]
 
 [project.urls]
@@ -33,4 +52,4 @@ Issues = "https://github.com/Rappsilber-Laboratory/xiFDR-py/issues"
 packages = ["xifdr"]
 
 [tool.setuptools_scm]
-write_to = "xifdr/_version.py"  # Auto-generates version info
+write_to = "xifdr/_version.py"

pyproject.toml

@@ -34,25 +34,35 @@ def test_column_prep():
         "start_pos_p1": [[1], [1, 2], [1, 2, 3]],
         "protein_p2": [['C'], ['E', 'E', 'E'], ['F', 'X']],
         "start_pos_p2": [[1], [1, 1, 2], [2, 1]],
-        "link_pos_p1": [1, 2, 3],
-        "link_pos_p2": [7, 8, 9],
+        "link_pos_p1": [7, 2, 3],
+        "link_pos_p2": [1, 8, 9],
         "sequence_p1": ['ABC', 'DEF', 'GHI'],
         "sequence_p2": ['ABC', 'AAA', 'DEF'],
         "decoy_p1": [False, False, True],
         "decoy_p2": [False, True, True],
         "score": [0.2, 1.2, 2.2],
         "fdr_group": ['between', 'self', 'between'],
         "decoy_class": ['TT', 'TD', 'DD'],
-        "cl_pos_p1": [[1], [2, 3], [3, 4, 5]],
-        "cl_pos_p2": [[7], [8, 8, 9], [10, 9]],
+        "cl_pos_p1": [[7], [2, 3], [3, 4, 5]],
+        "cl_pos_p2": [[1], [8, 8, 9], [10, 9]],
         "TT": [True, False, False],
         "TD": [False, True, False],
         "DD": [False, False, True],
         "coverage_p1": [0.5, 0.5, 0.5],
         "coverage_p2": [0.5, 0.5, 0.5],
         "protein_score_p1": [0.1, 0.6, 1.1],
-        "protein_score_p2": [0.1, 0.6, 1.1]
+        "protein_score_p2": [0.1, 0.6, 1.1],
+        "group_swapped": [True, False, False],
     })
 
     df_res = prepare_columns(df)
-    assert_frame_equal(df_res, df_expect.select(df_res.columns))
+    assert_frame_equal(
+        df_res.select(df_expect.columns).sort(df_expect.columns),
+        df_expect.sort(df_expect.columns)
+    )
+    # Check that the resulting state is constant
+    df_res = prepare_columns(df)
+    assert_frame_equal(
+        df_res.sort(df_expect.columns),
+        df_expect.select(df_res.columns).sort(df_expect.columns)
+    )

tests/test_column_prep.py

@@ -9,12 +9,18 @@
 from contextlib import closing
 from multiprocessing import get_context
 import multiprocessing.dummy as mp_dummy
+import sys
 from .fdr import full_fdr
 from .utils.column_preparation import prepare_columns
 from .utils.knee_finder import find_knees
 
 logger = logging.getLogger(__name__)
 
+def is_gil_enabled():
+    if hasattr(sys, "_is_gil_enabled"):
+        return sys._is_gil_enabled()
+    return True
+
 def boost(df: pl.DataFrame,
           csm_fdr: (float, float) = (0.0, 1.0),
           pep_fdr: (float, float) = (0.0, 1.0),
@@ -26,6 +32,7 @@ def boost(df: pl.DataFrame,
           boost_level: str = "ppi",
           boost_between: bool = True,
           method: str = "manhattan",
+          decoy_adjunct: str = "REV_",
           countdown: int = 3,
           points: int = 10,
           n_jobs: int = -1,
@@ -84,6 +91,7 @@ def boost(df: pl.DataFrame,
             neg_boost_cols=neg_boost_cols,
             boost_level=boost_level,
             boost_between=boost_between,
+            decoy_adjunct=decoy_adjunct,
             countdown=countdown,
             points=points,
             n_jobs=n_jobs,
@@ -102,11 +110,50 @@ def boost_manhattan(df: pl.DataFrame,
                     neg_boost_cols: list = None,
                     boost_level: str = "ppi",
                     boost_between: bool = True,
+                    decoy_adjunct: str = "REV_",
                     countdown: int = 3,
                     points: int = 10,
                     n_jobs: int = -1,
                     **kwargs):
-    df = prepare_columns(df)
+    """
+    Core Entry point for Manhattan optimization of FDR.
+
+    Parameters
+    ----------
+    df
+        Input CSM/PSM dataframe
+    csm_fdr
+        Range of CSM-level FDR cutoffs
+    pep_fdr
+        Range of peptide-level FDR cutoffs
+    prot_fdr
+        Range of protein-level FDR cutoffs
+    link_fdr
+        Range of link-level FDR cutoffs
+    ppi_fdr
+        Range of protein pair level (PPI) FDR cutoffs
+    boost_cols
+        Columns where a HIGHER value is better (e.g. scores)
+    neg_boost_cols
+        Columns where a LOWER value is better (e.g. Mass Error)
+    boost_level
+        The FDR level to optimize for ('csm', 'pep', 'prot', 'link', 'ppi')
+    boost_between
+        Optimize only for between-protein links
+    countdown
+        Number of iterations without improvement before stopping
+    points
+        Grid points for Manhattan search
+    n_jobs
+        Number of parallel jobs. -1 for all available cores or automatic detection based on memory.
+    kwargs
+        Other parameters to be passed to `full_fdr`
+
+    Returns
+    -------
+        Best parameters found for the optimization
+    """
+    df = prepare_columns(df, decoy_adjunct=decoy_adjunct)
     param_ranges = (
         csm_fdr,
         pep_fdr,
@@ -127,8 +174,11 @@ def boost_manhattan(df: pl.DataFrame,
         best_params += [maxi]
 
     # Figure out knee points for starting
-    df = prepare_columns(df)
-    knee_points = find_knees(df.filter(pl.col ('fdr_group') == 'between'), **kwargs)
+    knee_points = find_knees(
+        df.filter(pl.col ('fdr_group') == 'between'),
+        decoy_adjunct=decoy_adjunct,
+        **kwargs
+    )
     for i, p in enumerate(knee_points):
         best_params[i] = p
         # Clip to param max
@@ -161,6 +211,7 @@ def boost_manhattan(df: pl.DataFrame,
         neg_boost_cols=neg_boost_cols,
         boost_level=boost_level,
         boost_between=boost_between,
+        decoy_adjunct=decoy_adjunct,
         **kwargs
     )
 
@@ -173,11 +224,16 @@ def boost_manhattan(df: pl.DataFrame,
         n_jobs = max(max_mem_cpu, 1)
         n_jobs = min(n_jobs, os.cpu_count())
         logger.info(f"Using {n_jobs} CPUs based on available memory.")
-    if n_jobs == 1:
-        mp = mp_dummy
+    if not is_gil_enabled():
+        logger.info(f"Using {n_jobs} threads (GIL disabled).")
+        pool_obj = mp_dummy.Pool(n_jobs)
+    elif n_jobs == 1:
+        pool_obj = mp_dummy.Pool(1)
     else:
-        mp = get_context('spawn')
-    with closing(mp.Pool(n_jobs)) as pool:
+        # spawn is the only method that reliably works across all OSs with PyInstaller
+        pool_obj = get_context('spawn').Pool(processes=n_jobs)
+
+    with closing(pool_obj) as pool:
         while True:
             grids = []
             for param_index in range(n_params):
@@ -255,6 +311,7 @@ def boost_manhattan(df: pl.DataFrame,
 
 def _optimization_template(cutoffs,
                            df: pl.DataFrame,
+                           decoy_adjunct: str = 'REV_',
                            min_len: int = 5,
                            unique_csm: bool = True,
                            boost_cols: list = [],
@@ -263,7 +320,45 @@ def _optimization_template(cutoffs,
                            boost_between: bool = True,
                            td_prob: int = 2,
                            td_prot_prob: int = 10,
-                           td_dd_ratio: float = 1.0) -> float:
+                           td_dd_ratio: float = 1.0,
+                           custom_aggs: dict = None) -> float:
+    """
+    Template for parallel optimization and calculation of the score.
+
+    Parameters
+    ----------
+    cutoffs
+        A list of cutoffs for the different levels
+    df
+        The input CSM dataframe
+    decoy_adjunct
+        The prefix/suffix for decoy proteins
+    min_len
+        Minimum peptide length
+    unique_csm
+        Unique CSM aggregation
+    boost_cols
+        Columns to filter for HIGHER values
+    neg_boost_cols
+        Columns to filter for LOWER values
+    boost_level
+        The level to optimize for
+    boost_between
+        Optimize for between links
+    td_prob
+        Minimum threshold for TT/TD counts (except protein)
+    td_prot_prob
+        Minimum threshold for TT/TD counts on protein level
+    td_dd_ratio
+        Minimum ratio for matching DD/TD
+    custom_aggs
+        Custom aggregation expressions for the FDR levels
+
+    Returns
+    -------
+        Resulting score (negative estimated true positives)
+    """
+    df_height = df.height
     fdrs = cutoffs[:5]
     col_levels = cutoffs[5:]
     neg_col_levels = col_levels[len(boost_cols):]
@@ -285,12 +380,14 @@ def _optimization_template(cutoffs,
         )
     result_all = full_fdr(
         df, *fdrs,
+        decoy_adjunct=decoy_adjunct,
         min_len=min_len,
         unique_csm=unique_csm,
         prepare_column=False,
         td_prob=0,
         td_prot_prob=td_prot_prob,
-        td_dd_ratio=0
+        td_dd_ratio=0,
+        custom_aggs=custom_aggs
     )
     result = result_all[boost_level]
     if boost_between:
@@ -316,6 +413,6 @@ def _optimization_template(cutoffs,
             td_prob_bad = gl_tt*cutoffs[li] < td_prob
             dd_prob_bad = gl_dd*td_dd_ratio > gl_td
             if td_prob_bad or dd_prob_bad:
-                return -tp/df.height
+                return -tp/df_height
 
     return -tp