From 23f49ed384b77177314fb8feb7a7012549237a15 Mon Sep 17 00:00:00 2001 From: Ahmad lutfi Date: Fri, 28 Aug 2026 11:49:59 +0200 Subject: [PATCH 1/8] test --- .dockerignore | 14 + Dockerfile | 33 ++ README.md | 190 +++++- apptainer.def | 48 ++ cli_parser.py | 46 +- config.yml | 2 +- config_container_small.yml | 46 ++ gui.py | 560 +++++++++++++++++- requirements-cli.txt | 9 + runtime_metrics.py | 184 ++++++ sdrf_columns.py | 370 ++++++++++++ sdrf_export.py | 346 +++++++++++ .../test_sdrf_export.cpython-311.pyc | Bin 0 -> 11105 bytes tests/test_sdrf_export.py | 172 ++++++ 14 files changed, 1995 insertions(+), 25 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 apptainer.def create mode 100644 config_container_small.yml create mode 100644 requirements-cli.txt create mode 100644 runtime_metrics.py create mode 100644 sdrf_columns.py create mode 100644 sdrf_export.py create mode 100644 tests/__pycache__/test_sdrf_export.cpython-311.pyc create mode 100644 tests/test_sdrf_export.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2fdbaab --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.agents +.codex +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +build/ +dist/ +*.egg-info/ +output/ +*.sif +*.sqsh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2f9472b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3.11-slim-bullseye + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/opt/metaxtract +ENV PATH=/opt/venv/bin:$PATH + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + dirmngr \ + gnupg \ + && gpg --homedir /tmp --no-default-keyring \ + --keyring gnupg-ring:/usr/share/keyrings/mono-official-archive-keyring.gpg \ + --keyserver hkp://keyserver.ubuntu.com:80 \ + --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \ + && chmod +r /usr/share/keyrings/mono-official-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/debian stable-buster/snapshots/6.12.0.182 main" > /etc/apt/sources.list.d/mono-official-stable.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends mono-complete \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/metaxtract + +COPY requirements-cli.txt ./ +RUN python3 -m venv /opt/venv \ + && pip install --no-cache-dir --upgrade pip setuptools wheel \ + && pip install --no-cache-dir -r requirements-cli.txt + +COPY . . + +ENTRYPOINT ["python", "/opt/metaxtract/main.py"] +CMD ["--help"] diff --git a/README.md b/README.md index 0172f97..297d05f 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ MetaXtract is a hybrid tool for extracting, analysing, and visualising data from - Exports data as: - CSV / TSV - Parquet (peak lists) + - SDRF-Proteomics (`metadata.sdrf.tsv`) - Interactive Plotly HTML reports - Cross-sample visual comparisons - Designed for downstream computational analysis @@ -63,6 +64,167 @@ python main.py ```bash python main.py --config /path/to/config.yml ``` + +### Running with Docker or Apptainer/Singularity + +MetaXtract can also be run as a CLI-only Linux container. This is the recommended +mode for using macOS and for HPC infrastructure where a graphical +desktop is not available. + +Build the Docker image: + +```bash +docker build -t metaxtract:latest . +``` + +Run a mounted RAW-file directory and output directory: + +```bash +docker run --rm \ + -v /path/to/raw_files:/data:ro \ + -v /path/to/output:/out \ + metaxtract:latest \ + --input /data/sample.raw \ + --output-dir /out \ + --file-based-details \ + --complete-ms1 \ + --complete-ms2 \ + --ms1-peaklist-export \ + --ms2-peaklist-export \ + --graphical-representation +``` + +Example using the bundled small RAW file: + +```bash +mkdir -p output/docker_small +docker run --rm \ + -v "$(pwd)/data:/data:ro" \ + -v "$(pwd)/output/docker_small:/out" \ + metaxtract:latest \ + --input /data/small.RAW \ + --output-dir /out \ + --file-based-details \ + --complete-ms1 \ + --complete-ms2 \ + --ms1-peaklist-export \ + --ms2-peaklist-export \ + --graphical-representation +``` + +The repository also includes `config_container_small.yml`, a container-ready +configuration for the bundled small RAW file: + +```yaml +io: + input: + - /data/small.RAW + output_dir: /out + +outputs: + file_based_details: true + ms_method: false + lc_method: false + ms2_peaklist_export: true + ms1_peaklist_export: true + ms2_technical_details_export: false + ms1_technical_details_export: false + hdf5_export: false + +scan_header: + MS1: + select_all: true + columns: + Ion Injection Time (ms): true + Total Number of Peaks: true + Total Ion Current: true + Scan Start Time (min): true + Base Peak Intensity: true + Base Peak m/z: true + Scan Mode: true + thermo_Multi Inject Info: true + thermo_Multiple Injection: true + MS2: + select_all: true + columns: + Total Ion Current: true + Total Number of Peaks: true + Scan Start Time (min): true + Base Peak Intensity: true + Base Peak m/z: true + Selected Ion Intensity: true + Filter String: true + Scan Mode: true + +visualisation: + enabled: true + format: html + +multi_comparison: + enabled: false +``` + +Run Docker with the included config file: + +```bash +mkdir -p output/docker_small +docker run --rm \ + -v "$(pwd)/data:/data:ro" \ + -v "$(pwd)/output/docker_small:/out" \ + -v "$(pwd)/config_container_small.yml:/config.yml:ro" \ + metaxtract:latest \ + --config /config.yml +``` + +On Apple Silicon Macs, build and run the Linux x86_64 image explicitly if the +default platform does not work: + +```bash +docker build --platform linux/amd64 -t metaxtract:latest . +docker run --platform linux/amd64 --rm -v /path/to/raw_files:/data:ro -v /path/to/output:/out metaxtract:latest --input /data/sample.raw --output-dir /out --file-based-details +``` + +For HPC systems, build an Apptainer/Singularity image from Docker or from the +included definition file: + +```bash +apptainer build metaxtract.sif apptainer.def +apptainer run --bind /path/to/raw_files:/data,/path/to/output:/out metaxtract.sif --input /data/sample.raw --output-dir /out --file-based-details +``` + +Run the bundled small RAW file with Apptainer/Singularity: + +```bash +mkdir -p output/hpc_small +apptainer run \ + --bind "$(pwd)/data:/data","$(pwd)/output/hpc_small:/out" \ + metaxtract.sif \ + --input /data/small.RAW \ + --output-dir /out \ + --file-based-details \ + --complete-ms1 \ + --complete-ms2 \ + --ms1-peaklist-export \ + --ms2-peaklist-export \ + --graphical-representation +``` + +Run Apptainer/Singularity with the included config file: + +```bash +mkdir -p output/hpc_small +apptainer run \ + --bind "$(pwd)/data:/data","$(pwd)/output/hpc_small:/out","$(pwd)/config_container_small.yml:/config.yml" \ + metaxtract.sif \ + --config /config.yml +``` + +The container uses the bundled Thermo RawFileReader DLLs through Mono and +`pythonnet`. The GUI is not included in the container workflow; use command-line +options or a YAML configuration file. MS-method and LC-method export should be +disabled on Linux containers because those Thermo method-reading calls are not +supported on Linux. + #### Configuration File (`config.yml`) MetaXtract can be fully configured using a YAML configuration file. @@ -73,7 +235,9 @@ This allows reproducible, automated runs without passing long CLI arguments. ```yaml io: input: - - path/to/sample.RAW + - path/to/sample_1.RAW + - path/to/sample_2.RAW + - path/to/sample_3.RAW output_dir: output outputs: @@ -101,6 +265,11 @@ scan_header: visualisation: enabled: true format: html + +multi_comparison: + enabled: true + # Select any 2 or more inputs using their 1-based positions in io.input. + samples: [1, 2, 3] ``` --- ### GUI Options @@ -165,7 +334,17 @@ Columns intentionally marked as Thermo-specific because they are RAW trailer fie Check the [documentation](Doc/Doc.pdf) for more details. #### Visualisation -Interactive Plotly HTML reports, MS1 and MS2 trends, and Cross-sample overlays and boxplots. +Interactive Plotly HTML reports, MS1 and MS2 trends, and cross-sample overlays and boxplots. In the GUI, enable **Multi-sample comparison** and select any 2 or more of the loaded samples. In YAML/CLI runs, list the samples under `multi_comparison.samples` using 1-based indices such as `[1, 2, 4]`. + +#### Runtime and memory logging +For every processed RAW file, both the GUI log and CLI output report memory at the start and a final summary containing runtime, ending memory, sampled peak memory, and memory change. Memory is the resident set size (RSS) of the MetaXtract process, so it includes Python, native libraries, and Thermo/.NET allocations used while that file is processed. + +#### SDRF-Proteomics export +Enable **Export SDRF-Proteomics metadata (.sdrf.tsv)** in the GUI to open the metadata editor before processing. MetaXtract fills the RAW filename, instrument model, acquisition date, technology type, SDRF version, and template. Initially, the grid shows only required MS-proteomics inputs that cannot be determined reliably from a RAW file. + +Use **Add column** to search and multi-select from the complete known column-name catalog in the official SDRF templates registry. This includes sample, clinical, organism, DIA, single-cell, crosslinking, immunopeptidomics, metaproteomics, environmental, affinity-proteomics, and metabolomics fields. A custom `factor value[...]` column can also be added from the same picker. Added columns must be filled for every row and can be removed again with **Remove optional column**. + +The editor starts with one sample-to-file row per selected RAW file. Additional rows can be added for multiplexed experiments where multiple samples or labels share a RAW file. The dataset-level file is written as `metadata.sdrf.tsv` in the root output directory using the `ms-proteomics v1.1.0` template. --- ## Using MetaXtract as a Python Library @@ -268,10 +447,3 @@ This project is licensed under the Apache-2.0 license. **RawFileReader** reading tool. Copyright © 2016 by Thermo Fisher Scientific, Inc. All rights reserved. See [THERMO_LICENSE.txt](https://github.com/lutfia95/MetaXtract/blob/main/os_data/THERMO_LICENSE.txt) for licensing information. Note: anyone recieving RawFileReader as part of a larger software distribution (in the current context, as part of MetaXtract) is considered an "end user" under section 3.3 of the RawFileReader License, and is not granted rights to redistribute RawFileReader. - - - - - - - diff --git a/apptainer.def b/apptainer.def new file mode 100644 index 0000000..fd77326 --- /dev/null +++ b/apptainer.def @@ -0,0 +1,48 @@ +Bootstrap: docker +From: python:3.11-slim-bullseye + +%post + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg + gpg --homedir /tmp --no-default-keyring \ + --keyring gnupg-ring:/usr/share/keyrings/mono-official-archive-keyring.gpg \ + --keyserver hkp://keyserver.ubuntu.com:80 \ + --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + chmod +r /usr/share/keyrings/mono-official-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/debian stable-buster/snapshots/6.12.0.182 main" > /etc/apt/sources.list.d/mono-official-stable.list + apt-get update + apt-get install -y --no-install-recommends mono-complete + rm -rf /var/lib/apt/lists/* + python3 -m venv /opt/venv + /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel + /opt/venv/bin/pip install --no-cache-dir -r /opt/metaxtract/requirements-cli.txt + +%files + requirements-cli.txt /opt/metaxtract/requirements-cli.txt + LICENSE /opt/metaxtract/LICENSE + README.md /opt/metaxtract/README.md + config.yml /opt/metaxtract/config.yml + config_container_small.yml /opt/metaxtract/config_container_small.yml + main.py /opt/metaxtract/main.py + cli_parser.py /opt/metaxtract/cli_parser.py + raw_parser.py /opt/metaxtract/raw_parser.py + plotly_visualizer.py /opt/metaxtract/plotly_visualizer.py + runtime_metrics.py /opt/metaxtract/runtime_metrics.py + anndata_export.py /opt/metaxtract/anndata_export.py + explore_peak_list.py /opt/metaxtract/explore_peak_list.py + sdrf_columns.py /opt/metaxtract/sdrf_columns.py + sdrf_export.py /opt/metaxtract/sdrf_export.py + os_data /opt/metaxtract/os_data + assets /opt/metaxtract/assets + img /opt/metaxtract/img + data /opt/metaxtract/data + tests /opt/metaxtract/tests + +%environment + export PYTHONUNBUFFERED=1 + export PYTHONPATH=/opt/metaxtract + export PATH=/opt/venv/bin:$PATH + +%runscript + exec /opt/venv/bin/python /opt/metaxtract/main.py "$@" diff --git a/cli_parser.py b/cli_parser.py index 91ac3d1..c0142fa 100644 --- a/cli_parser.py +++ b/cli_parser.py @@ -17,6 +17,7 @@ write_comparison_html_multi, write_comparison_html_with_boxplots, ) +from runtime_metrics import FileUsageMonitor, format_bytes, format_file_usage def _cancel_requested(should_stop=None) -> bool: @@ -132,17 +133,19 @@ def _selected_columns(block: dict) -> list[str]: def _pick_cmp_inputs(all_inputs: list[str], samples_1based: list[int]) -> list[str]: - if not samples_1based or len(samples_1based) != 2: - raise ValueError("multi_comparison.samples must have exactly 2 indices (1-based), e.g. [1,3].") + if not isinstance(samples_1based, list) or len(samples_1based) < 2: + raise ValueError( + "multi_comparison.samples must have at least 2 indices (1-based), e.g. [1, 3, 4]." + ) out = [] for idx in samples_1based: - if not isinstance(idx, int): + if not isinstance(idx, int) or isinstance(idx, bool): raise ValueError("multi_comparison.samples must be integers.") if idx < 1 or idx > len(all_inputs): raise ValueError(f"multi_comparison index {idx} out of range for {len(all_inputs)} inputs.") out.append(all_inputs[idx - 1]) - if out[0] == out[1]: - raise ValueError("multi_comparison.samples must point to two different files.") + if len(set(samples_1based)) != len(samples_1based) or len(set(out)) != len(out): + raise ValueError("multi_comparison.samples must point to different files.") return out @@ -393,6 +396,27 @@ def write_info_tsv(raw_parser, out_tsv_path: str, should_stop=None): def run_cli(args): stop_event = threading.Event() + file_usage_monitor = None + file_usage_path = None + + def start_file_usage(input_file): + nonlocal file_usage_monitor, file_usage_path + file_usage_path = input_file + file_usage_monitor = FileUsageMonitor().start() + print( + f"[METRICS] Started: {input_file} | " + f"Memory RSS: {format_bytes(file_usage_monitor.start_rss_bytes)}" + ) + + def finish_file_usage(status): + nonlocal file_usage_monitor, file_usage_path + monitor = file_usage_monitor + input_file = file_usage_path + file_usage_monitor = None + file_usage_path = None + if monitor is None or input_file is None: + return + print(f"[METRICS] {status}: {input_file} | {format_file_usage(monitor.stop())}") def request_stop(signum, _frame): if stop_event.is_set(): @@ -485,7 +509,12 @@ def request_stop(signum, _frame): continue print(f"[INFO] Processing: {input_file}") - raw_parser = MetaXtract(input_file) + start_file_usage(input_file) + try: + raw_parser = MetaXtract(input_file) + except Exception: + finish_file_usage("Failed") + raise base = os.path.splitext(os.path.basename(input_file))[0] sample_out = os.path.join(outdir, base) @@ -623,9 +652,12 @@ def request_stop(signum, _frame): all_ms2_prec.append((base, x, y)) raw_parser.CloseRAWFile() - print(f"[INFO] Done: {input_file}\n") + print(f"[INFO] Done: {input_file}") + finish_file_usage("Finished") + print() if stop_event.is_set(): + finish_file_usage("Stopped") print("[INFO] Processing stopped.") if not stop_event.is_set() and graphical_representation and len(all_ms1_tic) >= 2: diff --git a/config.yml b/config.yml index b1d9ef8..6835aff 100644 --- a/config.yml +++ b/config.yml @@ -20,7 +20,7 @@ visualisation: multi_comparison: enabled: true - samples: [1, 2] # 1-based indices into io.input (file1 + file3) + samples: [1, 2, 3] # Select 2 or more samples using 1-based indices into io.input scan_header: MS1: diff --git a/config_container_small.yml b/config_container_small.yml new file mode 100644 index 0000000..b6947c4 --- /dev/null +++ b/config_container_small.yml @@ -0,0 +1,46 @@ +io: + input: + - /data/small.RAW + output_dir: /out + +outputs: + file_based_details: true + ms_method: false + lc_method: false + ms2_peaklist_export: true + ms1_peaklist_export: true + ms2_technical_details_export: false + ms1_technical_details_export: false + hdf5_export: false + +scan_header: + MS1: + select_all: true + columns: + Ion Injection Time (ms): true + Total Number of Peaks: true + Total Ion Current: true + Scan Start Time (min): true + Base Peak Intensity: true + Base Peak m/z: true + Scan Mode: true + thermo_Multi Inject Info: true + thermo_Multiple Injection: true + MS2: + select_all: true + columns: + Total Ion Current: true + Total Number of Peaks: true + Scan Start Time (min): true + Base Peak Intensity: true + Base Peak m/z: true + Selected Ion Intensity: true + Filter String: true + Scan Mode: true + +visualisation: + enabled: true + format: html + +multi_comparison: + enabled: false diff --git a/gui.py b/gui.py index f534bd5..7bfeff3 100644 --- a/gui.py +++ b/gui.py @@ -14,11 +14,15 @@ from PySide6.QtGui import QFont from PySide6.QtWidgets import ( QApplication, + QAbstractItemView, QCheckBox, + QComboBox, QDialog, QFileDialog, QGroupBox, + QHeaderView, QHBoxLayout, + QInputDialog, QLabel, QLineEdit, QMainWindow, @@ -29,6 +33,8 @@ QSizePolicy, QSpacerItem, QTextEdit, + QTableWidget, + QTableWidgetItem, QVBoxLayout, QWidget, QListWidget, @@ -45,6 +51,14 @@ ) from anndata_export import export_ms2_to_h5ad +from runtime_metrics import FileUsageMonitor, format_bytes, format_file_usage +from sdrf_columns import column_group +from sdrf_export import ( + available_sdrf_columns, + enrich_sdrf_rows_for_file, + validate_sdrf_metadata, + write_sdrf, +) class MS1Visualizer: def __init__(self, single_file_name: str, output_dir: str): @@ -260,6 +274,7 @@ def __init__( export_fmt: str | None, multi_cmp: bool, cmp_files: list[str] | None, + sdrf_payload: dict | None = None, hdf5_export: bool = False, ms2_peaklist_export: bool = False, ms1_peaklist_export: bool = False, @@ -282,8 +297,11 @@ def __init__( self.ms2_technical_details_export = bool(ms2_technical_details_export) self.ms1_technical_details_export = bool(ms1_technical_details_export) self.cmp_files = (cmp_files or []) + self.sdrf_payload = sdrf_payload or {} self._stop_event = threading.Event() self._current_raw_parser = None + self._file_usage_monitor: FileUsageMonitor | None = None + self._file_usage_path: str | None = None @Slot() def stop(self): @@ -298,6 +316,24 @@ def _close_current_raw_file(self) -> None: if raw_parser is not None: safe_call(lambda: raw_parser.CloseRAWFile(), None) + def _start_file_usage(self, selected_file: str) -> None: + self._file_usage_path = selected_file + self._file_usage_monitor = FileUsageMonitor().start() + self.log.emit( + f"[METRICS] Started: {selected_file} | " + f"Memory RSS: {format_bytes(self._file_usage_monitor.start_rss_bytes)}" + ) + + def _finish_file_usage(self, status: str) -> None: + monitor = self._file_usage_monitor + selected_file = self._file_usage_path + self._file_usage_monitor = None + self._file_usage_path = None + if monitor is None or selected_file is None: + return + usage = monitor.stop() + self.log.emit(f"[METRICS] {status}: {selected_file} | {format_file_usage(usage)}") + def _remove_empty_lines(self, input_file: str) -> None: try: with open(input_file, "r", encoding="utf-8", errors="replace") as f: @@ -641,6 +677,7 @@ def run(self): ms1_box_tic, ms1_box_bpi, ms1_box_tnp = {}, {}, {} ms2_box_tic, ms2_box_bpi, ms2_box_tnp = {}, {}, {} + sdrf_rows = [] global_out = Path(self.output_dir_raw) global_out.mkdir(parents=True, exist_ok=True) @@ -653,6 +690,7 @@ def run(self): break self.log.emit(f"[INFO] Processing: {selected_file}") + self._start_file_usage(selected_file) raw_parser = MetaXtract(selected_file) self._current_raw_parser = raw_parser @@ -660,6 +698,37 @@ def run(self): out_dir = Path(self.output_dir_raw) / base out_dir.mkdir(parents=True, exist_ok=True) + if self.sdrf_payload: + instrument_details = safe_call(lambda: raw_parser.GetInstrumentDetails(), {}) or {} + instrument_candidates = ( + instrument_details.get("Instrument Model"), + instrument_details.get("Instrument Name"), + safe_call(lambda: raw_parser.GetInstrumentName(), ""), + ) + instrument = next( + ( + str(value).strip() + for value in instrument_candidates + if value + and str(value).strip().casefold() + not in {"unknown", "n/a", "not available"} + ), + "", + ) + acquisition_date = safe_call(lambda: raw_parser.GetFileCreationDate(), "") + file_rows = enrich_sdrf_rows_for_file( + self.sdrf_payload.get("rows", []), + selected_file, + instrument, + acquisition_date, + ) + if any(not row.get("instrument") for row in file_rows): + raise ValueError( + f"No instrument model was found for {selected_file}. " + "Provide an instrument override in the SDRF editor." + ) + sdrf_rows.extend(file_rows) + plotly_ms1 = PlotlyMS1Visualizer(base, str(out_dir)) if self.plotly_enabled else None plotly_ms2 = PlotlyMS2Visualizer(base, str(out_dir)) if self.plotly_enabled else None @@ -810,10 +879,13 @@ def run(self): safe_call(lambda: raw_parser.CloseRAWFile(), None) self._current_raw_parser = None self.progress.emit(100) - self.log.emit(f"[INFO] Finished: {selected_file}\n") + self.log.emit(f"[INFO] Finished: {selected_file}") + self._finish_file_usage("Finished") + self.log.emit("") if self.should_stop(): self._close_current_raw_file() + self._finish_file_usage("Stopped") self.log.emit("[INFO] Processing stopped.") if not self.should_stop() and self.plotly_enabled and len(all_ms1_tic) >= 2: @@ -852,13 +924,24 @@ def run(self): ], ) self.log.emit(f"[VIS] MS2 comparison: {out}") + + if not self.should_stop() and self.sdrf_payload: + out = write_sdrf( + global_out / "metadata.sdrf.tsv", + sdrf_rows, + factor_name=self.sdrf_payload.get("factor_name", ""), + extra_columns=self.sdrf_payload.get("extra_columns", []), + ) + self.log.emit(f"[INFO] SDRF-Proteomics metadata: {out}") self.finished.emit() except InterruptedError: self._close_current_raw_file() + self._finish_file_usage("Stopped") self.log.emit("[INFO] Processing stopped.") self.finished.emit() except Exception as e: self._close_current_raw_file() + self._finish_file_usage("Failed") #self.failed.emit(str(e)) tb = traceback.format_exc() try: @@ -867,16 +950,16 @@ def run(self): pass self.failed.emit(f"{e}\n\n{tb}") -class TwoFilePickerDialog(QDialog): +class ComparisonSamplePickerDialog(QDialog): def __init__(self, files: list[str], parent=None): super().__init__(parent) - self.setWindowTitle("Select 2 files to compare") + self.setWindowTitle("Select samples to compare") self.setMinimumSize(760, 420) self._files = files self.selected: list[str] = [] lay = QVBoxLayout(self) - lay.addWidget(QLabel("Pick exactly 2 files:", self)) + lay.addWidget(QLabel("Select at least 2 samples for the comparison:", self)) self.listw = QListWidget(self) self.listw.setSelectionMode(QListWidget.MultiSelection) @@ -886,24 +969,473 @@ def __init__(self, files: list[str], parent=None): lay.addWidget(self.listw, 1) btns = QHBoxLayout() + btn_all = QPushButton("Select all", self) + btn_clear = QPushButton("Clear", self) btn_cancel = QPushButton("Cancel", self) btn_ok = QPushButton("OK", self) + btns.addWidget(btn_all) + btns.addWidget(btn_clear) btns.addStretch(1) btns.addWidget(btn_cancel) btns.addWidget(btn_ok) lay.addLayout(btns) + btn_all.clicked.connect(self.listw.selectAll) + btn_clear.clicked.connect(self.listw.clearSelection) btn_cancel.clicked.connect(self.reject) btn_ok.clicked.connect(self._accept_checked) def _accept_checked(self): picked = [it.text() for it in self.listw.selectedItems()] - if len(picked) != 2: - QMessageBox.critical(self, "Error", "Select exactly 2 files.") + if len(picked) < 2: + QMessageBox.critical(self, "Error", "Select at least 2 samples.") return self.selected = picked self.accept() + +class SdrfColumnPickerDialog(QDialog): + CUSTOM_FACTOR = "factor value[custom factor]" + + def __init__(self, headers: list[str], parent=None): + super().__init__(parent) + self.setWindowTitle("Add SDRF columns") + self.setMinimumSize(760, 580) + self.selected_headers: list[str] = [] + + layout = QVBoxLayout(self) + help_text = QLabel( + "Search the official SDRF registry, select one or more known column names, " + "then click Add selected columns.", + self, + ) + help_text.setWordWrap(True) + layout.addWidget(help_text) + + self.search = QLineEdit(self) + self.search.setPlaceholderText("Search, e.g. disease, treatment, collision energy...") + layout.addWidget(self.search) + + self.listw = QListWidget(self) + self.listw.setSelectionMode(QAbstractItemView.ExtendedSelection) + factor_item = QListWidgetItem( + f"{self.CUSTOM_FACTOR} — Experimental factors" + ) + factor_item.setData(Qt.UserRole, self.CUSTOM_FACTOR) + factor_item.setToolTip("Adds factor value[your factor name] after asking for its name.") + self.listw.addItem(factor_item) + for header in headers: + group = column_group(header) + item = QListWidgetItem(f"{header} — {group}") + item.setData(Qt.UserRole, header) + item.setToolTip(f"Official SDRF column: {header}") + self.listw.addItem(item) + layout.addWidget(self.listw, 1) + + self.count_label = QLabel(self) + layout.addWidget(self.count_label) + + buttons = QHBoxLayout() + btn_cancel = QPushButton("Cancel", self) + btn_add = QPushButton("Add selected columns", self) + buttons.addStretch(1) + buttons.addWidget(btn_cancel) + buttons.addWidget(btn_add) + layout.addLayout(buttons) + + self.search.textChanged.connect(self._filter_items) + btn_cancel.clicked.connect(self.reject) + btn_add.clicked.connect(self._accept_selected) + self._filter_items("") + + def _filter_items(self, query: str) -> None: + words = query.casefold().split() + visible = 0 + for index in range(self.listw.count()): + item = self.listw.item(index) + matches = all(word in item.text().casefold() for word in words) + item.setHidden(not matches) + if matches: + visible += 1 + self.count_label.setText(f"{visible} columns shown") + + def _accept_selected(self) -> None: + self.selected_headers = [ + item.data(Qt.UserRole) for item in self.listw.selectedItems() + ] + if not self.selected_headers: + QMessageBox.information(self, "SDRF columns", "Select at least one column.") + return + self.accept() + + +class SdrfMetadataDialog(QDialog): + REQUIRED_COLUMNS = [ + ("RAW file", "file", "comment[data file]", False), + ("source name", "source_name", "source name", False), + ("assay name", "assay_name", "assay name", False), + ("characteristics[organism]", "organism", "characteristics[organism]", False), + ("characteristics[organism part]", "organism_part", "characteristics[organism part]", False), + ("characteristics[biological replicate]", "biological_replicate", "characteristics[biological replicate]", False), + ("comment[proteomics data acquisition method]", "acquisition_method", "comment[proteomics data acquisition method]", False), + ("comment[label]", "label", "comment[label]", False), + ("comment[cleavage agent details]", "cleavage_agent", "comment[cleavage agent details]", False), + ("comment[fraction identifier]", "fraction_identifier", "comment[fraction identifier]", False), + ("comment[technical replicate]", "technical_replicate", "comment[technical replicate]", False), + ("comment[instrument] (auto / override)", "instrument_override", "comment[instrument]", False), + ] + + def __init__(self, files: list[str], parent=None): + super().__init__(parent) + self.setWindowTitle("SDRF-Proteomics metadata") + self.setMinimumSize(1050, 620) + self.files = list(files) + self.payload: dict = {} + self.columns = list(self.REQUIRED_COLUMNS) + self.setStyleSheet( + """ + QDialog { + background: #0f1115; + color: #e9eef5; + } + QLabel { + color: #e9eef5; + font-size: 12px; + } + QLineEdit, QComboBox { + background: #161a22; + color: #e9eef5; + border: 1px solid #364158; + border-radius: 7px; + padding: 6px 8px; + selection-background-color: #7a001a; + selection-color: #ffffff; + } + QLineEdit:focus, QComboBox:focus { + border: 1px solid #c21f45; + } + QLineEdit::placeholder { + color: #8fa0b6; + } + QComboBox::drop-down { + background: #202737; + border: none; + border-left: 1px solid #364158; + border-top-right-radius: 7px; + border-bottom-right-radius: 7px; + width: 24px; + } + QComboBox QAbstractItemView { + background: #161a22; + color: #e9eef5; + border: 1px solid #364158; + selection-background-color: #7a001a; + selection-color: #ffffff; + outline: none; + } + QTableWidget, QListWidget { + background: #121620; + alternate-background-color: #171c27; + color: #e9eef5; + gridline-color: #30394c; + border: 1px solid #364158; + border-radius: 9px; + selection-background-color: #7a001a; + selection-color: #ffffff; + outline: none; + } + QTableWidget::item, QListWidget::item { + color: #e9eef5; + padding: 6px; + border: none; + } + QTableWidget::item:selected, QListWidget::item:selected { + background: #7a001a; + color: #ffffff; + } + QHeaderView::section { + background: #202737; + color: #ffffff; + border: none; + border-right: 1px solid #364158; + border-bottom: 1px solid #48556f; + padding: 8px 6px; + font-weight: 700; + } + QTableCornerButton::section { + background: #202737; + border: none; + border-right: 1px solid #364158; + border-bottom: 1px solid #48556f; + } + QPushButton { + background: #7a001a; + color: #ffffff; + border: 1px solid #a00023; + border-radius: 10px; + padding: 8px 13px; + font-weight: 700; + } + QPushButton:hover { + background: #920020; + border-color: #c21f45; + } + QPushButton:pressed { + background: #600014; + } + QScrollBar:vertical, QScrollBar:horizontal { + background: #0b0d12; + border: none; + margin: 0; + } + QScrollBar::handle:vertical, QScrollBar::handle:horizontal { + background: #48556f; + border-radius: 5px; + min-height: 24px; + min-width: 24px; + } + QScrollBar::handle:vertical:hover, QScrollBar::handle:horizontal:hover { + background: #65738e; + } + QScrollBar::add-line, QScrollBar::sub-line, + QScrollBar::add-page, QScrollBar::sub-page { + background: transparent; + border: none; + } + QToolTip { + background: #202737; + color: #ffffff; + border: 1px solid #48556f; + padding: 5px; + } + """ + ) + + layout = QVBoxLayout(self) + instructions = QLabel( + "Only required MS-proteomics fields are shown initially. Complete them, then use " + "Add column for any optional, recommended, or specialized SDRF field. " + "MetaXtract fills the data filename, instrument, acquisition date, SDRF version, and template. " + "Use DDA/DIA/PRM/SRM and Trypsin/Lys-C shorthand if desired.", + self, + ) + instructions.setWordWrap(True) + layout.addWidget(instructions) + + self.table = QTableWidget(0, len(self.columns), self) + self.table.setHorizontalHeaderLabels([column[0] for column in self.columns]) + self.table.setAlternatingRowColors(True) + self.table.setShowGrid(True) + self.table.setSelectionBehavior(QAbstractItemView.SelectRows) + self.table.setSelectionMode(QAbstractItemView.ExtendedSelection) + self.table.verticalHeader().setVisible(False) + self.table.verticalHeader().setDefaultSectionSize(36) + header = self.table.horizontalHeader() + header.setSectionResizeMode(QHeaderView.Interactive) + header.setStretchLastSection(True) + self.table.setColumnWidth(0, 280) + self.table.setColumnWidth(1, 140) + self.table.setColumnWidth(2, 140) + for file_path in self.files: + self._append_row(self._defaults_for_file(file_path)) + layout.addWidget(self.table, 1) + + edit_buttons = QHBoxLayout() + btn_add = QPushButton("Add sample row", self) + btn_remove = QPushButton("Remove selected rows", self) + btn_copy = QPushButton("Copy selected metadata to all rows", self) + btn_add_column = QPushButton("Add column", self) + btn_remove_column = QPushButton("Remove optional column", self) + edit_buttons.addWidget(btn_add) + edit_buttons.addWidget(btn_remove) + edit_buttons.addWidget(btn_copy) + edit_buttons.addWidget(btn_add_column) + edit_buttons.addWidget(btn_remove_column) + edit_buttons.addStretch(1) + layout.addLayout(edit_buttons) + + dialog_buttons = QHBoxLayout() + btn_cancel = QPushButton("Cancel", self) + btn_export = QPushButton("Use this metadata", self) + dialog_buttons.addStretch(1) + dialog_buttons.addWidget(btn_cancel) + dialog_buttons.addWidget(btn_export) + layout.addLayout(dialog_buttons) + + btn_add.clicked.connect(self._add_sample_row) + btn_remove.clicked.connect(self._remove_selected_rows) + btn_copy.clicked.connect(self._copy_selected_metadata) + btn_add_column.clicked.connect(self._add_columns) + btn_remove_column.clicked.connect(self._remove_optional_column) + btn_cancel.clicked.connect(self.reject) + btn_export.clicked.connect(self._accept_metadata) + + def _defaults_for_file(self, file_path: str) -> dict: + base = os.path.splitext(os.path.basename(file_path))[0] + return { + "file": file_path, + "source_name": base, + "assay_name": base, + "organism": "", + "organism_part": "not available", + "biological_replicate": "1", + "acquisition_method": "", + "label": "", + "cleavage_agent": "", + "fraction_identifier": "1", + "technical_replicate": "1", + "instrument_override": "", + } + + def _append_row(self, values: dict) -> None: + row_index = self.table.rowCount() + self.table.insertRow(row_index) + for column_index, (_, key, _, _) in enumerate(self.columns): + value = str(values.get(key, "")) + if key == "file": + combo = QComboBox(self.table) + combo.addItems(self.files) + combo.setCurrentText(value) + self.table.setCellWidget(row_index, column_index, combo) + else: + self.table.setItem(row_index, column_index, QTableWidgetItem(value)) + + def _value(self, row_index: int, column_index: int) -> str: + widget = self.table.cellWidget(row_index, column_index) + if isinstance(widget, QComboBox): + return widget.currentText().strip() + item = self.table.item(row_index, column_index) + return item.text().strip() if item is not None else "" + + def _row_values(self, row_index: int) -> dict: + return { + key: self._value(row_index, column_index) + for column_index, (_, key, _, _) in enumerate(self.columns) + } + + def _add_sample_row(self) -> None: + source_row = self.table.currentRow() + if source_row >= 0: + values = self._row_values(source_row) + values["source_name"] = "" + values["label"] = "" + for _, key, header, removable in self.columns: + if removable and header.startswith("factor value["): + values[key] = "" + else: + values = self._defaults_for_file(self.files[0]) + self._append_row(values) + self.table.setCurrentCell(self.table.rowCount() - 1, 1) + + def _remove_selected_rows(self) -> None: + selected_rows = sorted({index.row() for index in self.table.selectedIndexes()}, reverse=True) + for row_index in selected_rows: + self.table.removeRow(row_index) + + def _copy_selected_metadata(self) -> None: + source_row = self.table.currentRow() + if source_row < 0: + QMessageBox.information(self, "SDRF metadata", "Select a row to copy first.") + return + values = self._row_values(source_row) + for row_index in range(self.table.rowCount()): + if row_index == source_row: + continue + for column_index in range(3, len(self.columns)): + key = self.columns[column_index][1] + item = self.table.item(row_index, column_index) + if item is None: + item = QTableWidgetItem() + self.table.setItem(row_index, column_index, item) + item.setText(values.get(key, "")) + + def _add_columns(self) -> None: + existing_headers = [column[2] for column in self.columns] + picker = SdrfColumnPickerDialog( + available_sdrf_columns(existing_headers), + self, + ) + if picker.exec() != QDialog.Accepted: + return + + for selected_header in picker.selected_headers: + header = selected_header + if header == SdrfColumnPickerDialog.CUSTOM_FACTOR: + factor_name, accepted = QInputDialog.getText( + self, + "Experimental factor", + "Factor name (for example: disease, treatment, time):", + ) + factor_name = factor_name.strip().casefold() + if not accepted: + continue + if not factor_name or any(char in factor_name for char in "[]\t\r\n"): + QMessageBox.critical( + self, + "Invalid factor name", + "Enter a factor name without brackets, tabs, or line breaks.", + ) + continue + header = f"factor value[{factor_name}]" + + if header in {column[2] for column in self.columns}: + QMessageBox.information( + self, + "SDRF columns", + f"{header} is already present.", + ) + continue + self._append_optional_column(header) + + def _append_optional_column(self, header: str) -> None: + column_index = self.table.columnCount() + self.table.insertColumn(column_index) + self.columns.append((header, header, header, True)) + header_item = QTableWidgetItem(header) + header_item.setToolTip(f"SDRF column: {header}") + self.table.setHorizontalHeaderItem(column_index, header_item) + self.table.setColumnWidth(column_index, max(180, min(360, len(header) * 8))) + for row_index in range(self.table.rowCount()): + self.table.setItem(row_index, column_index, QTableWidgetItem("")) + self.table.setCurrentCell(0, column_index) + + def _remove_optional_column(self) -> None: + column_index = self.table.currentColumn() + if column_index < 0: + QMessageBox.information(self, "SDRF columns", "Select a column first.") + return + label, _, header, removable = self.columns[column_index] + if not removable: + QMessageBox.information( + self, + "SDRF columns", + f"{label} is a required column and cannot be removed.", + ) + return + self.table.removeColumn(column_index) + self.columns.pop(column_index) + + def _accept_metadata(self) -> None: + rows = [self._row_values(row_index) for row_index in range(self.table.rowCount())] + extra_columns = [header for _, _, header, removable in self.columns if removable] + errors = validate_sdrf_metadata( + rows, + self.files, + extra_columns=extra_columns, + ) + if errors: + shown = errors[:12] + if len(errors) > len(shown): + shown.append(f"...and {len(errors) - len(shown)} more errors") + QMessageBox.critical(self, "Invalid SDRF metadata", "\n".join(shown)) + return + self.payload = { + "factor_name": "", + "extra_columns": extra_columns, + "rows": rows, + } + self.accept() + class MetaXtract_GUI(QMainWindow): def __init__(self): super().__init__() @@ -998,9 +1530,10 @@ def __init__(self): self.cb_ms1_peaklist = QCheckBox("Export MS1 peak list (parquet)", self) self.cb_ms2_technical_details = QCheckBox("Export MS2 technical details", self) self.cb_ms1_technical_details = QCheckBox("Export MS1 technical details", self) + self.cb_sdrf = QCheckBox("Export SDRF-Proteomics metadata (.sdrf.tsv)", self) #self.cb_hdf5 = QCheckBox("AnnData (HDF5 .h5ad) [MS2 only]", self) - self.cb_multi_cmp = QCheckBox("Multi sample comparison (2 files selection)", self) + self.cb_multi_cmp = QCheckBox("Multi-sample comparison (choose 2 or more samples)", self) self.cb_multi_cmp.setEnabled(True) opt.addWidget(self.cb_multi_cmp) opt.addWidget(self.cb_file_details) @@ -1011,6 +1544,7 @@ def __init__(self): opt.addWidget(self.cb_ms1_peaklist) opt.addWidget(self.cb_ms2_technical_details) opt.addWidget(self.cb_ms1_technical_details) + opt.addWidget(self.cb_sdrf) #opt.addWidget(self.cb_hdf5) main.addWidget(gb_opt) @@ -1203,6 +1737,7 @@ def extract_information(self): ms1_peaklist_export = self.cb_ms1_peaklist.isChecked() ms2_technical_details_export = self.cb_ms2_technical_details.isChecked() ms1_technical_details_export = self.cb_ms1_technical_details.isChecked() + sdrf_enabled = self.cb_sdrf.isChecked() #hdf5_export = self.cb_hdf5.isChecked() hdf5_export = False export_fmt = None @@ -1224,11 +1759,19 @@ def extract_information(self): if len(selected_files) == 2: cmp_files = list(selected_files) else: - dlg = TwoFilePickerDialog(list(selected_files), self) + dlg = ComparisonSamplePickerDialog(list(selected_files), self) if dlg.exec() != QDialog.Accepted: return cmp_files = dlg.selected + sdrf_payload = None + if sdrf_enabled: + processed_files = cmp_files if (multi_cmp and cmp_files) else selected_files + dlg = SdrfMetadataDialog(list(processed_files), self) + if dlg.exec() != QDialog.Accepted: + return + sdrf_payload = dlg.payload + if self._thread is not None and self._thread.isRunning(): self.show_error("Processing already running.") return @@ -1254,6 +1797,7 @@ def extract_information(self): export_fmt=export_fmt, multi_cmp=multi_cmp, cmp_files=cmp_files, + sdrf_payload=sdrf_payload, hdf5_export=hdf5_export, ms2_peaklist_export=ms2_peaklist_export, ms1_peaklist_export=ms1_peaklist_export, diff --git a/requirements-cli.txt b/requirements-cli.txt new file mode 100644 index 0000000..545e673 --- /dev/null +++ b/requirements-cli.txt @@ -0,0 +1,9 @@ +numpy +pandas +pyarrow +pyyaml +tqdm +plotly +anndata +h5py +pythonnet diff --git a/runtime_metrics.py b/runtime_metrics.py new file mode 100644 index 0000000..aa1edc2 --- /dev/null +++ b/runtime_metrics.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import os +import sys +import threading +import time +from dataclasses import dataclass + + +if sys.platform == "win32": + import ctypes + from ctypes import wintypes + + class _ProcessMemoryCounters(ctypes.Structure): + _fields_ = [ + ("cb", wintypes.DWORD), + ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t), + ] + + +def _windows_rss_bytes() -> int | None: + if sys.platform != "win32": + return None + try: + counters = _ProcessMemoryCounters() + counters.cb = ctypes.sizeof(counters) + get_current_process = ctypes.windll.kernel32.GetCurrentProcess + get_current_process.restype = wintypes.HANDLE + get_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo + get_memory_info.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(_ProcessMemoryCounters), + wintypes.DWORD, + ] + get_memory_info.restype = wintypes.BOOL + process = get_current_process() + ok = get_memory_info( + process, + ctypes.byref(counters), + counters.cb, + ) + return int(counters.WorkingSetSize) if ok else None + except Exception: + return None + + +def _linux_rss_bytes() -> int | None: + try: + with open("/proc/self/status", "r", encoding="ascii") as status_file: + for line in status_file: + if line.startswith("VmRSS:"): + return int(line.split()[1]) * 1024 + except (OSError, ValueError, IndexError): + pass + + try: + with open("/proc/self/statm", "r", encoding="ascii") as statm_file: + resident_pages = int(statm_file.read().split()[1]) + return resident_pages * int(os.sysconf("SC_PAGE_SIZE")) + except (OSError, ValueError, IndexError): + return None + + +def get_process_rss_bytes() -> int | None: + """Return this process's resident memory, including native/.NET allocations.""" + if sys.platform == "win32": + return _windows_rss_bytes() + if sys.platform.startswith("linux"): + return _linux_rss_bytes() + + # Optional fallback for other platforms. psutil is not required by MetaXtract. + try: + import psutil + + return int(psutil.Process(os.getpid()).memory_info().rss) + except Exception: + return None + + +@dataclass(frozen=True) +class FileUsage: + elapsed_seconds: float + start_rss_bytes: int | None + end_rss_bytes: int | None + peak_rss_bytes: int | None + + +class FileUsageMonitor: + """Sample process RSS while one input file is being processed.""" + + def __init__(self, sample_interval_seconds: float = 0.2): + self.sample_interval_seconds = max(0.05, float(sample_interval_seconds)) + self.started_at: float | None = None + self.start_rss_bytes: int | None = None + self.peak_rss_bytes: int | None = None + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + self._result: FileUsage | None = None + + def start(self) -> FileUsageMonitor: + if self.started_at is not None: + return self + self.started_at = time.perf_counter() + self.start_rss_bytes = get_process_rss_bytes() + self.peak_rss_bytes = self.start_rss_bytes + self._thread = threading.Thread( + target=self._sample_until_stopped, + name="metaxtract-memory-monitor", + daemon=True, + ) + self._thread.start() + return self + + def _sample(self) -> int | None: + rss_bytes = get_process_rss_bytes() + if rss_bytes is not None and ( + self.peak_rss_bytes is None or rss_bytes > self.peak_rss_bytes + ): + self.peak_rss_bytes = rss_bytes + return rss_bytes + + def _sample_until_stopped(self) -> None: + while not self._stop_event.wait(self.sample_interval_seconds): + self._sample() + + def stop(self) -> FileUsage: + if self._result is not None: + return self._result + if self.started_at is None: + self.start() + + self._stop_event.set() + if self._thread is not None and self._thread is not threading.current_thread(): + self._thread.join(timeout=self.sample_interval_seconds * 2) + + end_rss_bytes = self._sample() + elapsed_seconds = time.perf_counter() - float(self.started_at) + self._result = FileUsage( + elapsed_seconds=elapsed_seconds, + start_rss_bytes=self.start_rss_bytes, + end_rss_bytes=end_rss_bytes, + peak_rss_bytes=self.peak_rss_bytes, + ) + return self._result + + +def format_bytes(byte_count: int | None) -> str: + if byte_count is None: + return "unavailable" + return f"{byte_count / (1024 * 1024):.1f} MiB" + + +def format_duration(seconds: float) -> str: + hours, remainder = divmod(max(0.0, seconds), 3600) + minutes, remaining_seconds = divmod(remainder, 60) + if hours >= 1: + return f"{int(hours):02d}:{int(minutes):02d}:{remaining_seconds:05.2f}" + if minutes >= 1: + return f"{int(minutes):02d}:{remaining_seconds:05.2f}" + return f"{remaining_seconds:.2f} s" + + +def format_file_usage(usage: FileUsage) -> str: + runtime = f"Runtime: {format_duration(usage.elapsed_seconds)}" + if usage.start_rss_bytes is None or usage.end_rss_bytes is None: + return f"{runtime} | Memory RSS: unavailable" + + change_bytes = usage.end_rss_bytes - usage.start_rss_bytes + change_sign = "+" if change_bytes >= 0 else "-" + change = format_bytes(abs(change_bytes)) + return ( + f"{runtime} | Memory RSS (start -> end): " + f"{format_bytes(usage.start_rss_bytes)} -> {format_bytes(usage.end_rss_bytes)}" + f" | Peak: {format_bytes(usage.peak_rss_bytes)}" + f" | Change: {change_sign}{change}" + ) diff --git a/sdrf_columns.py b/sdrf_columns.py new file mode 100644 index 0000000..92144af --- /dev/null +++ b/sdrf_columns.py @@ -0,0 +1,370 @@ +"""Known SDRF column headers exposed by the metadata editor. + +This is a local snapshot of the unique column names in the official BigBio +``sdrf-templates`` registry (retrieved 2026-08-27). Keeping the names local +makes the picker complete and usable without network access. +""" + +from __future__ import annotations + + +KNOWN_SDRF_HEADERS = tuple(filter(None, r""" +assay name +characteristics[age] +characteristics[alkalinity method] +characteristics[alkalinity] +characteristics[alkyl diethers] +characteristics[altitude] +characteristics[aluminum saturation method] +characteristics[aluminum saturation] +characteristics[aminopeptidase activity] +characteristics[ammonium] +characteristics[analyte class] +characteristics[ancestry category] +characteristics[ann arbor stage] +characteristics[antibiotic treatment] +characteristics[antibody enrichment] +characteristics[atmospheric data] +characteristics[bacterial carbon production] +characteristics[bacterial respiration] +characteristics[biological replicate] +characteristics[biomass estimation] +characteristics[biomass] +characteristics[biopsy site] +characteristics[biorepository] +characteristics[biosample accession number] +characteristics[bishomohopanol] +characteristics[body mass index] +characteristics[broad-scale environmental context] +characteristics[bromide] +characteristics[calcium] +characteristics[carbon nitrogen ratio] +characteristics[cell cycle phase] +characteristics[cell diameter] +characteristics[cell identifier] +characteristics[cell line authentication] +characteristics[cell line] +characteristics[cell type] +characteristics[cell viability] +characteristics[cellosaurus accession] +characteristics[cellosaurus name] +characteristics[cells per well] +characteristics[chemical administration] +characteristics[chloride] +characteristics[chlorophyll] +characteristics[clinical data] +characteristics[clinical history] +characteristics[collection date] +characteristics[compound] +characteristics[conductivity] +characteristics[crop rotation] +characteristics[crosslink distance] +characteristics[crosslinking reaction time] +characteristics[crosslinking temperature] +characteristics[culture medium] +characteristics[current land use] +characteristics[current vegetation] +characteristics[density] +characteristics[depletion] +characteristics[depth] +characteristics[developmental stage] +characteristics[diether lipids] +characteristics[disease staging] +characteristics[disease] +characteristics[dissolved carbon dioxide] +characteristics[dissolved hydrogen] +characteristics[dissolved inorganic carbon] +characteristics[dissolved inorganic nitrogen] +characteristics[dissolved inorganic phosphorus] +characteristics[dissolved organic carbon] +characteristics[dissolved organic nitrogen] +characteristics[dissolved oxygen] +characteristics[dose] +characteristics[downward PAR] +characteristics[drainage classification] +characteristics[dukes stage] +characteristics[elevation] +characteristics[enrichment marker] +characteristics[enrichment process] +characteristics[environmental medium] +characteristics[ethnicity] +characteristics[exposure duration] +characteristics[extreme unusual properties of heavy metals method] +characteristics[extreme unusual properties of heavy metals] +characteristics[fluorescence] +characteristics[forward scatter] +characteristics[gastrointestinal tract disorder] +characteristics[genetic modification] +characteristics[genotype] +characteristics[geographic location] +characteristics[gleason score] +characteristics[glucosidase activity] +characteristics[growth condition] +characteristics[height] +characteristics[histologic subtype] +characteristics[history of agrochemical additions] +characteristics[history of extreme event] +characteristics[history of fire] +characteristics[history of flooding] +characteristics[history of previous land use] +characteristics[history of tillage] +characteristics[host age] +characteristics[host body product] +characteristics[host body site] +characteristics[host body temperature] +characteristics[host body-mass index] +characteristics[host contamination] +characteristics[host diet] +characteristics[host disease status] +characteristics[host family relationship] +characteristics[host genotype] +characteristics[host height] +characteristics[host last meal] +characteristics[host occupation] +characteristics[host organism] +characteristics[host phenotype] +characteristics[host pulse] +characteristics[host sex] +characteristics[host subject id] +characteristics[host total mass] +characteristics[ihmc medication code] +characteristics[immunopeptidome enrichment method] +characteristics[individual] +characteristics[last follow up] +characteristics[light intensity] +characteristics[link classification information] +characteristics[link climate information] +characteristics[liver disorder] +characteristics[local environmental context] +characteristics[magnesium] +characteristics[material type] +characteristics[mean annual precipitation] +characteristics[mean annual temperature] +characteristics[mean friction velocity] +characteristics[mean peak friction velocity] +characteristics[mean seasonal precipitation] +characteristics[mean seasonal temperature] +characteristics[medical history performed] +characteristics[menopausal status] +characteristics[metastasis site] +characteristics[mhc protein complex] +characteristics[mhc typing method] +characteristics[mhc typing] +characteristics[microbiome source] +characteristics[mitotic rate] +characteristics[mock community composition] +characteristics[mock community] +characteristics[n-alkanes] +characteristics[nitrate] +characteristics[nitrite] +characteristics[nitrogen] +characteristics[observed host symbionts] +characteristics[organic carbon] +characteristics[organic matter] +characteristics[organic nitrogen] +characteristics[organism part] +characteristics[organism] +characteristics[pH method] +characteristics[particulate organic carbon] +characteristics[particulate organic nitrogen] +characteristics[passage number] +characteristics[perturbation] +characteristics[petroleum hydrocarbon] +characteristics[ph] +characteristics[phaeopigments] +characteristics[phenotype] +characteristics[phosphate] +characteristics[phospholipid fatty acid] +characteristics[photon flux] +characteristics[pooled sample] +characteristics[pooling of DNA extracts] +characteristics[potassium] +characteristics[pre-existing condition] +characteristics[pressure] +characteristics[primary production] +characteristics[profile position] +characteristics[project name] +characteristics[redox potential] +characteristics[salinity] +characteristics[sample collection method] +characteristics[sample matrix] +characteristics[sample name] +characteristics[sample storage temperature] +characteristics[sample storage] +characteristics[sample type] +characteristics[sampling depth zone] +characteristics[sampling site] +characteristics[sampling time] +characteristics[sex] +characteristics[side scatter] +characteristics[sieving] +characteristics[silicate] +characteristics[single cell isolation protocol] +characteristics[slope aspect] +characteristics[slope gradient] +characteristics[smoking status] +characteristics[sodium] +characteristics[soil horizon] +characteristics[soil local classification method] +characteristics[soil taxonomic local classification] +characteristics[soil taxonomic of FAO classification] +characteristics[soil texture method] +characteristics[soil texture] +characteristics[soil type method] +characteristics[soil type] +characteristics[soil water content method] +characteristics[soluble reactive phosphorus] +characteristics[spatial coordinates] +characteristics[special diet] +characteristics[spiked compound] +characteristics[strain or breed] +characteristics[sulfate] +characteristics[sulfide] +characteristics[survival time] +characteristics[suspended particulate matter] +characteristics[synthetic peptide] +characteristics[temperature] +characteristics[tidal stage] +characteristics[tissue mass] +characteristics[tissue supergroup] +characteristics[total depth of water volume] +characteristics[total dissolved nitrogen] +characteristics[total inorganic nitrogen] +characteristics[total nitrogen concentration] +characteristics[total nitrogen content method] +characteristics[total nitrogen content] +characteristics[total organic carbon method] +characteristics[total organic carbon] +characteristics[total particulate carbon] +characteristics[total phosphorus] +characteristics[treatment response] +characteristics[treatment status] +characteristics[treatment] +characteristics[tumor grading] +characteristics[tumor mass] +characteristics[tumor size] +characteristics[tumor stage] +characteristics[turbidity] +characteristics[water content] +characteristics[water current] +characteristics[weight] +characteristics[weiss grade] +comment[acquisition date] +comment[acquisition method] +comment[alkylation reagent] +comment[bacterial production] +comment[carrier channel] +comment[carrier gas] +comment[chemical cross-linking coupled with ms] +comment[chromatography column] +comment[chromatography type] +comment[cleavage agent details] +comment[collision energy] +comment[contaminant database] +comment[cross-linker] +comment[crosslink enrichment method] +comment[crosslinker concentration] +comment[crosslinker to protein ratio] +comment[current vegetation method] +comment[data file] +comment[derivatization agent] +comment[derivatization] +comment[dilution] +comment[dissociation method] +comment[elution conditions] +comment[expected organism list] +comment[extraction method] +comment[extraction solvent] +comment[facs nozzle size] +comment[facs sorting mode] +comment[flow rate] +comment[fraction identifier] +comment[fractionation method] +comment[fragment mass tolerance] +comment[gc column] +comment[gradient] +comment[history of previous land use method] +comment[horizon method] +comment[instrument] +comment[internal standard] +comment[ion source] +comment[isolation window width] +comment[label] +comment[lc batch] +comment[lcm microscope model] +comment[link to additional analysis] +comment[lot number] +comment[metabolite assignment file md5] +comment[metabolite assignment file] +comment[metagenome accession] +comment[microbial biomass method] +comment[microbial biomass] +comment[microfluidics chip type] +comment[miscellaneous parameter] +comment[mobile phase a] +comment[mobile phase b] +comment[modification parameters] +comment[ms max charge] +comment[ms max im] +comment[ms max mz] +comment[ms max rt] +comment[ms min charge] +comment[ms min im] +comment[ms min mz] +comment[ms min rt] +comment[ms1 scan range] +comment[ms2 mass analyzer] +comment[ms2 max mz] +comment[ms2 min mz] +comment[ms2 scan range] +comment[ms3 max mz] +comment[ms3 min mz] +comment[ms3 scan range] +comment[nanopots chip version] +comment[normalization method] +comment[organism count] +comment[oven program] +comment[oxygenation status of sample] +comment[panel name] +comment[plate] +comment[platform] +comment[precursor mass tolerance] +comment[processed data file md5] +comment[processed data file] +comment[proteomics data acquisition method] +comment[quantification unit] +comment[quenching reagent] +comment[raw data file md5] +comment[reduction reagent] +comment[reference channel] +comment[sample preparation batch] +comment[sample storage duration] +comment[sample storage location] +comment[sample storage temperature] +comment[sample volume or weight for DNA extraction] +comment[scan polarity] +comment[scan window lower limit] +comment[scan window upper limit] +comment[sdrf annotation tool] +comment[sdrf template] +comment[sdrf validation hash] +comment[sdrf version] +comment[size-fraction lower threshold] +comment[size-fraction upper threshold] +comment[storage conditions] +comment[technical replicate] +comment[tissue section] +source name +technology type +""".splitlines())) + + +def column_group(header: str) -> str: + if header.startswith("characteristics["): + return "Characteristics" + if header.startswith("comment["): + return "Comments" + if header.startswith("factor value["): + return "Experimental factors" + return "Core SDRF" + diff --git a/sdrf_export.py b/sdrf_export.py new file mode 100644 index 0000000..bdb3e3a --- /dev/null +++ b/sdrf_export.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import csv +import re +from datetime import datetime +from pathlib import Path + +from sdrf_columns import KNOWN_SDRF_HEADERS + + +SDRF_VERSION = "v1.1.0" +SDRF_TEMPLATE = "ms-proteomics v1.1.0" +TECHNOLOGY_TYPE = "proteomic profiling by mass spectrometry" + +ACQUISITION_METHODS = { + "dda": "NT=Data-dependent acquisition;AC=PRIDE:0000627", + "data-dependent acquisition": "NT=Data-dependent acquisition;AC=PRIDE:0000627", + "dia": "NT=Data-independent acquisition;AC=PRIDE:0000450", + "data-independent acquisition": "NT=Data-independent acquisition;AC=PRIDE:0000450", + "prm": "NT=Parallel reaction monitoring;AC=PRIDE:0000629", + "parallel reaction monitoring": "NT=Parallel reaction monitoring;AC=PRIDE:0000629", + "srm": "NT=Selected reaction monitoring;AC=PRIDE:0000630", + "selected reaction monitoring": "NT=Selected reaction monitoring;AC=PRIDE:0000630", +} + +CLEAVAGE_AGENTS = { + "trypsin": "NT=Trypsin;AC=MS:1001251", + "lys-c": "NT=Lys-C;AC=MS:1001309", + "lysc": "NT=Lys-C;AC=MS:1001309", + "chymotrypsin": "NT=Chymotrypsin;AC=MS:1001306", +} + +USER_REQUIRED_FIELDS = ( + ("source_name", "source name"), + ("assay_name", "assay name"), + ("organism", "characteristics[organism]"), + ("organism_part", "characteristics[organism part]"), + ("biological_replicate", "characteristics[biological replicate]"), + ("acquisition_method", "comment[proteomics data acquisition method]"), + ("label", "comment[label]"), + ("cleavage_agent", "comment[cleavage agent details]"), + ("fraction_identifier", "comment[fraction identifier]"), + ("technical_replicate", "comment[technical replicate]"), +) + +REQUIRED_SDRF_HEADERS = { + "source name", + "assay name", + "technology type", + "characteristics[organism]", + "characteristics[organism part]", + "characteristics[biological replicate]", + "comment[proteomics data acquisition method]", + "comment[instrument]", + "comment[cleavage agent details]", + "comment[label]", + "comment[fraction identifier]", + "comment[technical replicate]", + "comment[data file]", +} + +AUTOMATIC_SDRF_HEADERS = { + "technology type", + "comment[data file]", + "comment[acquisition date]", + "comment[sdrf version]", + "comment[sdrf template]", +} + +FIXED_SDRF_HEADERS = REQUIRED_SDRF_HEADERS | AUTOMATIC_SDRF_HEADERS +KNOWN_SDRF_HEADER_SET = set(KNOWN_SDRF_HEADERS) +KNOWN_SDRF_HEADERS_BY_CASEFOLD = { + header.casefold(): header for header in KNOWN_SDRF_HEADERS +} + + +def _text(value) -> str: + if value is None: + return "" + return str(value).replace("\t", " ").replace("\r", " ").replace("\n", " ").strip() + + +def _known_text(value) -> str: + cleaned = _text(value) + if cleaned.casefold() in {"", "unknown", "n/a", "not available"}: + return "" + return cleaned + + +def normalize_acquisition_method(value: str) -> str: + cleaned = _text(value) + return ACQUISITION_METHODS.get(cleaned.casefold(), cleaned) + + +def normalize_cleavage_agent(value: str) -> str: + cleaned = _text(value) + return CLEAVAGE_AGENTS.get(cleaned.casefold(), cleaned) + + +def normalize_acquisition_date(value) -> str: + cleaned = _text(value) + if not cleaned: + return "not available" + try: + return datetime.fromisoformat(cleaned).isoformat(timespec="seconds") + except ValueError: + pass + + for date_format in ( + "%m/%d/%Y %H:%M:%S", + "%m/%d/%Y %I:%M:%S %p", + "%d/%m/%Y %H:%M:%S", + "%Y-%m-%d %H:%M:%S", + ): + try: + return datetime.strptime(cleaned, date_format).isoformat(timespec="seconds") + except ValueError: + continue + return "not available" + + +def available_sdrf_columns(existing_headers=()) -> list[str]: + """Return official registry columns that are not already in the editor.""" + existing = {_text(header).casefold() for header in existing_headers} + return [ + header + for header in KNOWN_SDRF_HEADERS + if header.casefold() not in existing + and header not in FIXED_SDRF_HEADERS + ] + + +def _normalized_extra_columns(extra_columns) -> list[str]: + normalized = [] + seen = set() + for value in extra_columns or []: + cleaned = _text(value) + folded = cleaned.casefold() + header = KNOWN_SDRF_HEADERS_BY_CASEFOLD.get(folded, folded) + if not header or folded in seen: + continue + normalized.append(header) + seen.add(folded) + return normalized + + +def _valid_factor_header(header: str) -> bool: + return bool(re.fullmatch(r"factor value\[[^\[\]\t\r\n]+\]", header)) + + +def validate_sdrf_metadata( + rows: list[dict], + selected_files: list[str], + factor_name: str = "", + extra_columns: list[str] | None = None, +) -> list[str]: + errors = [] + if not rows: + return ["Add at least one sample-to-file row."] + + selected = set(map(str, selected_files)) + represented = set() + uniqueness_keys = set() + extra_headers = _normalized_extra_columns(extra_columns) + legacy_factor_header = "" + if factor_name: + legacy_factor_header = f"factor value[{_text(factor_name).casefold()}]" + if legacy_factor_header not in extra_headers: + extra_headers.append(legacy_factor_header) + + for header in extra_headers: + if header in FIXED_SDRF_HEADERS: + errors.append(f"{header} is already a required or automatically filled column.") + elif header not in KNOWN_SDRF_HEADER_SET and not _valid_factor_header(header): + errors.append(f"{header} is not a known SDRF column name.") + + for row_number, row in enumerate(rows, start=1): + file_path = _text(row.get("file")) + if file_path not in selected: + errors.append(f"Row {row_number}: select one of the loaded RAW files.") + else: + represented.add(file_path) + + for field, label in USER_REQUIRED_FIELDS: + if not _text(row.get(field)): + errors.append(f"Row {row_number}: {label} is required.") + + for field, label in ( + ("source_name", "source name"), + ("assay_name", "assay name"), + ("organism", "characteristics[organism]"), + ("acquisition_method", "comment[proteomics data acquisition method]"), + ("label", "comment[label]"), + ("cleavage_agent", "comment[cleavage agent details]"), + ): + if _text(row.get(field)).casefold() == "not available": + errors.append(f"Row {row_number}: {label} cannot be 'not available'.") + + biological_replicate = _text(row.get("biological_replicate")) + if biological_replicate != "pooled" and not _positive_integer(biological_replicate): + errors.append(f"Row {row_number}: biological replicate must be a positive integer or 'pooled'.") + for field, label in ( + ("fraction_identifier", "fraction identifier"), + ("technical_replicate", "technical replicate"), + ): + if not _positive_integer(_text(row.get(field))): + errors.append(f"Row {row_number}: {label} must be a positive integer.") + + for header in extra_headers: + value = row.get("factor_value") if header == legacy_factor_header else row.get(header) + if not _text(value): + errors.append(f"Row {row_number}: {header} cannot be empty once added.") + + has_dynamic_factor = any(_valid_factor_header(header) for header in extra_headers) + if not factor_name and not has_dynamic_factor and _text(row.get("factor_value")): + errors.append( + f"Row {row_number}: enter a factor name or clear the factor value." + ) + + uniqueness_key = tuple( + _text(row.get(field)) for field in ("source_name", "assay_name", "label") + ) + if uniqueness_key in uniqueness_keys: + errors.append( + f"Row {row_number}: source name + assay name + label must be unique." + ) + uniqueness_keys.add(uniqueness_key) + + for missing_file in sorted(selected - represented): + errors.append(f"No SDRF row is assigned to {missing_file}.") + + if factor_name and not re.fullmatch(r"[^\[\]\t\r\n]+", factor_name): + errors.append("Factor name cannot contain brackets, tabs, or line breaks.") + + return errors + + +def _positive_integer(value: str) -> bool: + try: + return int(value) >= 1 and str(int(value)) == value + except (TypeError, ValueError): + return False + + +def enrich_sdrf_rows_for_file( + user_rows: list[dict], + file_path: str, + instrument: str, + acquisition_date, +) -> list[dict]: + enriched = [] + for user_row in user_rows: + if _text(user_row.get("file")) != str(file_path): + continue + row = dict(user_row) + row["file"] = str(file_path) + row["data_file"] = Path(file_path).name + row["instrument"] = _known_text(row.get("instrument_override")) or _known_text(instrument) + row["acquisition_date"] = normalize_acquisition_date(acquisition_date) + row["acquisition_method"] = normalize_acquisition_method(row.get("acquisition_method", "")) + row["cleavage_agent"] = normalize_cleavage_agent(row.get("cleavage_agent", "")) + enriched.append(row) + return enriched + + +def write_sdrf( + output_path: str | Path, + rows: list[dict], + factor_name: str = "", + extra_columns: list[str] | None = None, +) -> Path: + factor_name = _text(factor_name).casefold() + extra_headers = _normalized_extra_columns(extra_columns) + legacy_factor_header = f"factor value[{factor_name}]" if factor_name else "" + if legacy_factor_header and legacy_factor_header not in extra_headers: + extra_headers.append(legacy_factor_header) + + characteristic_headers = [ + header for header in extra_headers if header.startswith("characteristics[") + ] + factor_headers = [header for header in extra_headers if _valid_factor_header(header)] + other_headers = [ + header + for header in extra_headers + if header not in characteristic_headers and header not in factor_headers + ] + + headers = [ + "source name", + "characteristics[organism]", + "characteristics[organism part]", + "characteristics[biological replicate]", + *characteristic_headers, + "assay name", + "technology type", + "comment[proteomics data acquisition method]", + "comment[label]", + "comment[instrument]", + "comment[cleavage agent details]", + "comment[fraction identifier]", + "comment[technical replicate]", + "comment[data file]", + "comment[acquisition date]", + "comment[sdrf version]", + "comment[sdrf template]", + *other_headers, + *factor_headers, + ] + + output_rows = [] + for row in rows: + fixed_values = { + "source name": row.get("source_name"), + "characteristics[organism]": row.get("organism"), + "characteristics[organism part]": row.get("organism_part"), + "characteristics[biological replicate]": row.get("biological_replicate"), + "assay name": row.get("assay_name"), + "technology type": TECHNOLOGY_TYPE, + "comment[proteomics data acquisition method]": row.get("acquisition_method"), + "comment[label]": row.get("label"), + "comment[instrument]": row.get("instrument"), + "comment[cleavage agent details]": row.get("cleavage_agent"), + "comment[fraction identifier]": row.get("fraction_identifier"), + "comment[technical replicate]": row.get("technical_replicate"), + "comment[data file]": row.get("data_file"), + "comment[acquisition date]": row.get("acquisition_date") or "not available", + "comment[sdrf version]": SDRF_VERSION, + "comment[sdrf template]": SDRF_TEMPLATE, + } + values = [] + for header in headers: + if header in fixed_values: + values.append(fixed_values[header]) + elif header == legacy_factor_header: + values.append(row.get(header) or row.get("factor_value")) + else: + values.append(row.get(header)) + output_rows.append([_text(value) for value in values]) + + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", newline="", encoding="utf-8") as output_file: + writer = csv.writer(output_file, delimiter="\t", lineterminator="\n") + writer.writerow(headers) + writer.writerows(output_rows) + return output_path diff --git a/tests/__pycache__/test_sdrf_export.cpython-311.pyc b/tests/__pycache__/test_sdrf_export.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..409cadc16e17890bd36ffca0605b793baeeb6876 GIT binary patch literal 11105 zcmd5?Yit|GcHZT4Ns)R}rX)M2lh`tC>uLERzjAHUR%F?f9Li~;#9XNPT5QKlCFljEe$wgImBqf@0TQz{CPB3iMC&PljC}5c;F% z%#y2JN{XcM)$ZE0J^K5l0*&7O8-oa4@nYuuG_kGnJ6IG6E^dsxQC3^KgqGlq8x>?PV8 zwtL^sFkivH9>!|~AMg6ahHs?7#z0AbIg7auk*_K*4D`ju=zrM#9i~RZtmYU^NK;{_ z){?lKNTm}Wr-iu4lZkjTm(FLhqSh*8Nh&#|Cz9M1F+Pza@rhJg(Au&&l1Zdf1tFeD zewt5-DJhl9#`%OKXf2l^17mu{j36c8Eund@kW@J_X$%Y^Z29HI8Hn6r1O}NjZsXa{ zm~lIA1M1-IK%K~raTo6d>gHWQIo=J_!*f8rya#9v?*;1PYhp~;r`5$^I;XBq=ZG{W zh?1DBbcH?E!#{GevQ8{-o3s^~IOfq%WG+>ruuibd<-j|5C-36jJjZ)@FJHs^;Mc?Z z;l1`=t)*6-sa8GTKx;KZ*(RvfT&~rKwP?@A=y(Q(ABYwjx{-0+$y`<-BMOl37DT&p?gn%9WAJV^@iY$79Q-h?P7uIX_uMpg-|8kQ+>?B`ohgCI{Y{aoqGSx|r52_q?;c_xz=;yGpO^E^Xdb z+P1UQ{zhr%?$VCkrPe*A*LRmXcF`wX3hi9nQs=K-WRSeRM%}QfZrD_}%T)J`Z2>O~ zz%v7XFJN4ZWt+Jnfo`)2h}>a{%q;w_43eSL7$|dzsx&MGaU+Lk=WSL6ILX|Hl3y96 zQKiDxu}2ARJXoTd^1`Fch8X|~3JmuotVE%^dU;+^p&vUq3IrC8^F8H)Pq^SS-|MseUk=Ri^Oqi_WZ$Ub8&!Ry zC134xr`_Xy$pBfj60!$cuyEJ~36#TEK@{&WlA%T@_l!55H7O$-Z=ZKm@shpHN|nP4 znk=#w-aF@A*7DauS2HBg!Bh}=H%FfVWJUAQ1`J_IjS z&rh9fO*9vZMcz3bf&=9bHOaSdlI8^+l-mav@^{b8!5sS_UiHv9K5%IcK9!Vobx9Q` z>P%_{fIb}~c|o%$vezEiNf^`JWx3K^x}1tA1fjB|Y8_NjzFHg$2Y#d+AHq5Q1CSfc zv!+dV9rrx5+wS@1$Rkm18djQy)uv&&@$8KISwqL1d!hO3w#ThzD3!YpZrxs%tC{0#cEU^YHOy7BpgTkyj*Wbu-a1f2?FjT>X7 zFo_5Ni-^F3-9WAPG(@TSxtf-Zz^p8qCn~o`9}4p3o`qBL<^g!AdZ>tyBgl+jU|q}p z8Yu6tjSsCb^s~+KI1cL*AeKG&aI4%jq%;kwO+%0Sza5Y-y!-7Dx$(VqY(?b2qsu^) z_NWRp7*&H&g&R@15xMfL+&CT>0P^NVEzmp6By%0D8c5JXgt@o}>=<^|*cg=@{Iys^ zCOfnU*p#>3cYqB8|9YslVQzLx*mpTxGVH4Rkg^QR_c7P)k^$MFM>Ousp=h_Z$=eHH zR~1>_G4HIh#2iJ(ELW*#{^ecsZfh&Erco`Iw2tb!6C4xpXH_38ldXA!>19O6728$& z`^*)vPTmKrh1zT0<+he%5?VmzP;^?y!E@EEgVpD&ls5nJo}!cYE~~3`*Qh0<2-P<$p9omIUtS4Mc!5r8fS0!kZ;J%|qi^fSp zW}5mj8pj}(BMEXX0ty=a0@n!c9)cs*Jm6X3;}X=@ydroiq;x7Ph@=Pe>}esZaXN+^ z$^uNs{IE;6d1`WbX*w^((K(=buqSYWNK<;paXtlZ7rg{}VW3G74|TAKAn1T|3su{? z&rJ}ic}_iMB6SZGI#*TAYZX5Yjf?*RY7M;v!I_#8_y*b*8O{TMAIQv5$+zjL@3kks z*PhjfOU=RCL#5^|rRLqG=J0cmv!U@N1LV2g?XP{w09iE1e>XCv*#7AL*+VTt^+MK|L1^)v{Vw?S&ZH^P$-QVnF zfm-k+$`d{7{tO~(8;NlK;bx(s+9zN=)9|dE#THitIEBpK1d}nzJ6F{;F5Rfp+!wN+ zWOG-tWC+^Syvx2iikriZ#%q2%0u~O1}D=u$$^{p1Jku&2uGRpyb;ITc{blH$Y~% zMV}#XcY}aK+Gjt6rC&Iqa0gWGfLwW2+AgTO=z3yT>+_uPto0NDD+ShiHUAj$ztDsl zzO!ebv#)2YqUV9z$aI3zw8AI$CS|ta`xoQVlt41McoN;wl?#AE|EdgrMXeO{C!#yV z;vL%|cN|fCM^)cZ!%l*O1E)WA!4A?345E8t--QhFN0?i7 z&_;98P`0w~-R1iz%eLQ6B{4TKArQEh%BRzDs#nT)S8=#sNWg_dVc)8v_^N6-7>f8y zAh5w3cPfp$)W%&iZmQ6?KH;|hoqcxjH@+`@b4i8kQ@K8w>oZP)o`nvDJEU@lWbTlW z*t>8*;SQ_ZVVOH@B=#);?mwb(M`Z2@Z5sW-a%<<@l~-y84-AcJhW~-xji`oa{}#Nr zfNsDiYe6^10biIjcr~WD?kK_y+p?`&gqt=5l{_E{-ia~DcUkh?G}Sh1oFP!i2t|ig z|A7DU#9BMUjE{;qZrrmp6xn&sif~r?`C+b3a~(i(9oJn&=Pc^26~Th5Y(*Dby?L!? z-HLJVlp2IRa3k6qGuItw|ZIU2FkiumMvqLu$@f80LT=OrA-#N zL9g{tdtKC#3YIIl_V$FLCr{DxWEh|$kZ!$C^UV+|B+@(z+15(8_HF?_v%#IT1kA_HNYeJ;g zcCQAJS~fooY=08iK6gb9Y*zvY)WCtKffG*xCzL=$4MZNDR|A7j1LvLu&OPRpKuit9 z09gTQ8l$MGLJ>8Mv51-~DF}+zj+ar>ss#`=>By*OVTT#x=!v}x`&S-&-3u)SegX&u z55+OxymkD|NVom>-Hym!_wV+s8!jkZg5ojYn_6KWNNg)jC1K_dVz4;z3PCSw-N zK6qQcX#K)=voXfd+cfZ}quv4=>j3-^0>o@zEqRZCxhLVB_p!prs_K9Ux_=p*Dblq# zd|XjMPzj!=Zg2V#k?Bg&Xz(}K8D4tAePUg={@hLeh2vK23(swjk)->$snTw%*pu{5 zAOyB!I;1PBiv*1|3uAC)5lF1-UbRpMUwUAx9}6KmlOMbUi2gjBs9eh#n>!Z*^5*^UQuV9^i5r{jCY^uORMu}EbuRuPCp}j^>6qLedjyl-_Yfw13|^`p zI%$9pcpB;-ZS24p-qsSl%>)Y|jB%QO0%F$NklpkIT%WWS4*!G&p2e>J5(pqzD%P`O zU%n60tkiX@b=@-#xNU^9ih2r4?fc{dF}ZCFlz*ca{9}#aEdVm(f};@>ZTb2Um;cde zAWHkN3N$#Z28R{ytje90xwD8Q$_0^SkD*{pX^*KugE2K2Q@Ame8Mvrw7ua3d%z%PSO>lv#OHJl!9ixCwRYzyq?o8T=p2C*6YY09 z4}zPrLK= Date: Fri, 28 Aug 2026 12:23:47 +0200 Subject: [PATCH 2/8] Mchip --- README.md | 28 +++++++++++++++++--- cli_parser.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 297d05f..1ff1b7a 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,18 @@ python main.py --config /path/to/config.yml ### Running with Docker or Apptainer/Singularity MetaXtract can also be run as a CLI-only Linux container. This is the recommended -mode for using macOS and for HPC infrastructure where a graphical -desktop is not available. +mode for macOS and HPC infrastructure where a graphical desktop is not available. + +On macOS, `brew install docker` installs mostly the Docker CLI, not the Docker +daemon/engine. Mac computers run Linux containers through Docker Desktop or +another VM-based backend because macOS cannot run Linux containers directly. +Install Docker Desktop, open it, and wait until it finishes starting: + +```bash +brew install --cask docker +open -a Docker +docker info +``` Build the Docker image: @@ -181,7 +191,19 @@ default platform does not work: ```bash docker build --platform linux/amd64 -t metaxtract:latest . -docker run --platform linux/amd64 --rm -v /path/to/raw_files:/data:ro -v /path/to/output:/out metaxtract:latest --input /data/sample.raw --output-dir /out --file-based-details +mkdir -p output/docker_small +docker run --platform linux/amd64 --rm \ + -v "$(pwd)/data:/data:ro" \ + -v "$(pwd)/output/docker_small:/out" \ + metaxtract:latest \ + --input /data/small.RAW \ + --output-dir /out \ + --file-based-details \ + --complete-ms1 \ + --complete-ms2 \ + --ms1-peaklist-export \ + --ms2-peaklist-export \ + --graphical-representation ``` For HPC systems, build an Apptainer/Singularity image from Docker or from the diff --git a/cli_parser.py b/cli_parser.py index c0142fa..447b1fa 100644 --- a/cli_parser.py +++ b/cli_parser.py @@ -72,6 +72,69 @@ def _cancel_requested(should_stop=None) -> bool: "thermo_Multiple Injection": ("Multiple Injection",), } +DEFAULT_MS1_COLUMNS = [ + "Ion Injection Time (ms)", + "Total Number of Peaks", + "Total Ion Current", + "Scan Start Time (min)", + "Base Peak Intensity", + "Base Peak m/z", + "Scan Mode", + "thermo_Multi Inject Info", + "thermo_Multiple Injection", +] + +DEFAULT_MS2_COLUMNS = [ + "Total Ion Current", + "Total Number of Peaks", + "thermo_Number of Channels", + "Sampling Frequency", + "Collision Energy", + "Scan Start Time (min)", + "Scan Window m/z Range", + "Selected Ion Intensity", + "Filter String", + "Scan Mode", + "thermo_AGC", + "thermo_Micro Scan Count", + "Ion Injection Time (ms)", + "thermo_Elapsed Scan Time (sec)", + "Dissociation Method", + "Mass Analyzer Type", + "Detector Type", + "Base Peak m/z", + "thermo_Average Scan by Inst", + "thermo_Orbitrap Resolution", + "thermo_API Process Delay", + "thermo_Dependency Type", + "thermo_Multi Inject Info", + "Base Peak Intensity", + "thermo_Master Scan Number", + "Experimental Precursor Monoisotopic m/z", + "Charge State", + "Normalized Collision Energy (%)", + "Collision Energy (eV)", + "Isolation Window Width (m/z)", + "thermo_Access ID", + "thermo_Conversion Parameter I", + "thermo_Conversion Parameter A", + "thermo_Conversion Parameter B", + "thermo_Conversion Parameter C", + "thermo_Conversion Parameter D", + "thermo_Conversion Parameter E", + "thermo_Temperature Comp. (ppm)", + "thermo_RF Comp. (ppm)", + "thermo_Space Charge Comp. (ppm)", + "thermo_Resolution Comp. (ppm)", + "thermo_Number of LM Found", + "thermo_LM Correction (ppm)", + "thermo_RawOvFtT", + "thermo_Injection t0", + "thermo_Reagent Ion Injection Time (ms)", + "thermo_FAIMS Voltage On", + "FAIMS Compensation Voltage", +] + def trailer_value(trailer_data, output_label: str): if not trailer_data: @@ -476,9 +539,15 @@ def request_stop(signum, _frame): ms2_block = _cfg_get(cfg, ["scan_header", "MS2"], {}) or {} if getattr(args, "complete_ms1", False): - ms1_block = {"select_all": True, "columns": (ms1_block.get("columns", {}) or {})} + columns = ms1_block.get("columns", {}) if isinstance(ms1_block, dict) else {} + if not columns: + columns = {column: True for column in DEFAULT_MS1_COLUMNS} + ms1_block = {"select_all": True, "columns": columns} if getattr(args, "complete_ms2", False): - ms2_block = {"select_all": True, "columns": (ms2_block.get("columns", {}) or {})} + columns = ms2_block.get("columns", {}) if isinstance(ms2_block, dict) else {} + if not columns: + columns = {column: True for column in DEFAULT_MS2_COLUMNS} + ms2_block = {"select_all": True, "columns": columns} selected_ms1_options = _selected_columns(ms1_block) selected_ms2_options = _selected_columns(ms2_block) From 19b1afba7f3d94e0ad3b18df1c802ee7c97605cf Mon Sep 17 00:00:00 2001 From: Ahmad lutfi Date: Fri, 28 Aug 2026 12:29:01 +0200 Subject: [PATCH 3/8] fix --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1ff1b7a..9c23489 100644 --- a/README.md +++ b/README.md @@ -186,13 +186,13 @@ docker run --rm \ --config /config.yml ``` -On Apple Silicon Macs, build and run the Linux x86_64 image explicitly if the -default platform does not work: +On Apple Silicon Macs, build and run the image without `--platform` first so +Docker uses the native ARM64 Linux backend: ```bash -docker build --platform linux/amd64 -t metaxtract:latest . +docker build -t metaxtract:latest . mkdir -p output/docker_small -docker run --platform linux/amd64 --rm \ +docker run --rm \ -v "$(pwd)/data:/data:ro" \ -v "$(pwd)/output/docker_small:/out" \ metaxtract:latest \ @@ -206,6 +206,15 @@ docker run --platform linux/amd64 --rm \ --graphical-representation ``` +Do not use smart quotes copied from rich-text editors in Docker commands; use +plain shell quotes such as `"$(pwd)/data:/data:ro"`. + +Running the container with `--platform linux/amd64` on Apple Silicon uses +emulation. If Mono or `pythonnet` crashes with a message such as +`Assertion: should not be reached at tramp-amd64.c`, rebuild and run without +`--platform`. If native ARM64 execution does not work on the machine, run the +container on an x86_64 Linux workstation or HPC node instead. + For HPC systems, build an Apptainer/Singularity image from Docker or from the included definition file: From 86a4f6569db225fa325be9713d32026d746819d6 Mon Sep 17 00:00:00 2001 From: Ahmad lutfi Date: Fri, 28 Aug 2026 12:48:26 +0200 Subject: [PATCH 4/8] apptainer --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c23489..997d40f 100644 --- a/README.md +++ b/README.md @@ -218,12 +218,37 @@ container on an x86_64 Linux workstation or HPC node instead. For HPC systems, build an Apptainer/Singularity image from Docker or from the included definition file: +On WSL2 Ubuntu, Apptainer can be installed with: + +```bash +sudo apt update +sudo apt install -y software-properties-common +sudo add-apt-repository -y ppa:apptainer/ppa +sudo apt update +sudo apt install -y apptainer +``` + +Check the installation: + +```bash +apptainer --version +``` + +If the PPA is not available for the Ubuntu version installed in WSL2, check the +Ubuntu version with: + +```bash +lsb_release -a +``` + +Then from the MetaXtract repository, build the Apptainer/Singularity image: + ```bash apptainer build metaxtract.sif apptainer.def apptainer run --bind /path/to/raw_files:/data,/path/to/output:/out metaxtract.sif --input /data/sample.raw --output-dir /out --file-based-details ``` -Run the bundled small RAW file with Apptainer/Singularity: +Run the bundled small RAW file with Apptainer/Singularity on WSL2 or HPC: ```bash mkdir -p output/hpc_small From e0b7edd7e1b6d031a876970e9bddfdcfe8f85260 Mon Sep 17 00:00:00 2001 From: Ahmad lutfi Date: Fri, 28 Aug 2026 14:42:24 +0200 Subject: [PATCH 5/8] workflow_fix --- .dockerignore | 2 - .gitignore | 12 ++ README.md | 2 +- cli_parser.py | 18 +- config.yml | 7 +- gui.py | 15 +- runtime_metrics.py | 54 +++++ workflow/README.md | 175 ++++++++++++---- workflow/Snakefile | 275 ++++++++------------------ workflow/config.yaml | 57 +++++- workflow/requirements.txt | 1 + workflow/scripts/prepare_inputs.py | 132 +++++++++++++ workflow/scripts/run_metaxtract.py | 99 ++++++++++ workflow/scripts/summarize_runtime.py | 39 ++++ 14 files changed, 646 insertions(+), 242 deletions(-) create mode 100644 .gitignore create mode 100644 workflow/requirements.txt create mode 100644 workflow/scripts/prepare_inputs.py create mode 100644 workflow/scripts/run_metaxtract.py create mode 100644 workflow/scripts/summarize_runtime.py diff --git a/.dockerignore b/.dockerignore index 2fdbaab..b8562cd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,4 @@ .git -.agents -.codex __pycache__/ *.py[cod] .pytest_cache/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..18bf77f --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +output/ +workflow/results/ +workflow/.snakemake/ + +*.sif +mono_crash.*.json diff --git a/README.md b/README.md index 997d40f..802cd37 100644 --- a/README.md +++ b/README.md @@ -393,7 +393,7 @@ Check the [documentation](Doc/Doc.pdf) for more details. Interactive Plotly HTML reports, MS1 and MS2 trends, and cross-sample overlays and boxplots. In the GUI, enable **Multi-sample comparison** and select any 2 or more of the loaded samples. In YAML/CLI runs, list the samples under `multi_comparison.samples` using 1-based indices such as `[1, 2, 4]`. #### Runtime and memory logging -For every processed RAW file, both the GUI log and CLI output report memory at the start and a final summary containing runtime, ending memory, sampled peak memory, and memory change. Memory is the resident set size (RSS) of the MetaXtract process, so it includes Python, native libraries, and Thermo/.NET allocations used while that file is processed. +For every processed RAW file, both the GUI log and CLI output report memory at the start and a final summary containing runtime, ending memory, sampled peak memory, and memory change. Each run also writes `runtime_summary_YYYYMMDD_HHMMSS.tsv` in the root output directory. The TSV contains one row per processed RAW file with status, runtime in seconds, start/end/peak memory in GB, and memory change in GB. Memory is the resident set size (RSS) of the MetaXtract process, so it includes Python, native libraries, and Thermo/.NET allocations used while that file is processed. #### SDRF-Proteomics export Enable **Export SDRF-Proteomics metadata (.sdrf.tsv)** in the GUI to open the metadata editor before processing. MetaXtract fills the RAW filename, instrument model, acquisition date, technology type, SDRF version, and template. Initially, the grid shows only required MS-proteomics inputs that cannot be determined reliably from a RAW file. diff --git a/cli_parser.py b/cli_parser.py index 447b1fa..82521eb 100644 --- a/cli_parser.py +++ b/cli_parser.py @@ -17,7 +17,12 @@ write_comparison_html_multi, write_comparison_html_with_boxplots, ) -from runtime_metrics import FileUsageMonitor, format_bytes, format_file_usage +from runtime_metrics import ( + FileUsageMonitor, + append_runtime_usage_tsv, + format_bytes, + format_file_usage, +) def _cancel_requested(should_stop=None) -> bool: @@ -461,6 +466,7 @@ def run_cli(args): stop_event = threading.Event() file_usage_monitor = None file_usage_path = None + runtime_log_path = None def start_file_usage(input_file): nonlocal file_usage_monitor, file_usage_path @@ -479,7 +485,13 @@ def finish_file_usage(status): file_usage_path = None if monitor is None or input_file is None: return - print(f"[METRICS] {status}: {input_file} | {format_file_usage(monitor.stop())}") + usage = monitor.stop() + print(f"[METRICS] {status}: {input_file} | {format_file_usage(usage)}") + if runtime_log_path is not None: + try: + append_runtime_usage_tsv(runtime_log_path, input_file, status, usage) + except Exception as e: + print(f"[METRICS][WARN] Could not write runtime TSV: {e}") def request_stop(signum, _frame): if stop_event.is_set(): @@ -513,6 +525,8 @@ def request_stop(signum, _frame): sys.exit(1) os.makedirs(outdir, exist_ok=True) + runtime_log_path = Path(outdir) / f"runtime_summary_{datetime.now().strftime('%Y%m%d_%H%M%S')}.tsv" + print(f"[METRICS] Runtime TSV: {runtime_log_path}") cfg_outputs = _cfg_get(cfg, ["outputs"], {}) or {} hdf5_export = bool(getattr(args, "hdf5_export", False) or cfg_outputs.get("hdf5_export", False)) diff --git a/config.yml b/config.yml index 6835aff..d2695ab 100644 --- a/config.yml +++ b/config.yml @@ -1,9 +1,8 @@ io: input: - - "C:/Users/yynk1/Desktop/thermo_reader/github/MetaXtract/data/MS2_MS1_orbitrap.raw" - - "C:/Users/yynk1/Desktop/thermo_reader/github/MetaXtract/data/small.raw" - - "C:/Users/yynk1/Desktop/thermo_reader/data/B240103_02_Astral_ZC_QC_51_Hela.raw" - output_dir: "C:/Users/yynk1/Desktop/thermo_reader/github/MetaXtract/output/cli/" + - "data/MS2_MS1_orbitrap.raw" + - "data/small.raw" + output_dir: "output/cli/" outputs: file_based_details: true diff --git a/gui.py b/gui.py index 7bfeff3..9357b22 100644 --- a/gui.py +++ b/gui.py @@ -51,7 +51,12 @@ ) from anndata_export import export_ms2_to_h5ad -from runtime_metrics import FileUsageMonitor, format_bytes, format_file_usage +from runtime_metrics import ( + FileUsageMonitor, + append_runtime_usage_tsv, + format_bytes, + format_file_usage, +) from sdrf_columns import column_group from sdrf_export import ( available_sdrf_columns, @@ -302,6 +307,7 @@ def __init__( self._current_raw_parser = None self._file_usage_monitor: FileUsageMonitor | None = None self._file_usage_path: str | None = None + self._runtime_log_path: Path | None = None @Slot() def stop(self): @@ -333,6 +339,11 @@ def _finish_file_usage(self, status: str) -> None: return usage = monitor.stop() self.log.emit(f"[METRICS] {status}: {selected_file} | {format_file_usage(usage)}") + if self._runtime_log_path is not None: + try: + append_runtime_usage_tsv(self._runtime_log_path, selected_file, status, usage) + except Exception as e: + self.log.emit(f"[METRICS][WARN] Could not write runtime TSV: {e}") def _remove_empty_lines(self, input_file: str) -> None: try: @@ -681,6 +692,8 @@ def run(self): global_out = Path(self.output_dir_raw) global_out.mkdir(parents=True, exist_ok=True) + self._runtime_log_path = global_out / f"runtime_summary_{datetime.now().strftime('%Y%m%d_%H%M%S')}.tsv" + self.log.emit(f"[METRICS] Runtime TSV: {self._runtime_log_path}") cmp_set = set(self.cmp_files) if (self.multi_cmp and self.cmp_files) else None for selected_file in self.selected_files: diff --git a/runtime_metrics.py b/runtime_metrics.py index aa1edc2..3179568 100644 --- a/runtime_metrics.py +++ b/runtime_metrics.py @@ -4,7 +4,9 @@ import sys import threading import time +import csv from dataclasses import dataclass +from pathlib import Path if sys.platform == "win32": @@ -158,6 +160,12 @@ def format_bytes(byte_count: int | None) -> str: return f"{byte_count / (1024 * 1024):.1f} MiB" +def bytes_to_gb(byte_count: int | None) -> str: + if byte_count is None: + return "" + return f"{byte_count / (1024 ** 3):.6f}" + + def format_duration(seconds: float) -> str: hours, remainder = divmod(max(0.0, seconds), 3600) minutes, remaining_seconds = divmod(remainder, 60) @@ -182,3 +190,49 @@ def format_file_usage(usage: FileUsage) -> str: f" | Peak: {format_bytes(usage.peak_rss_bytes)}" f" | Change: {change_sign}{change}" ) + + +RUNTIME_TSV_HEADERS = [ + "raw_file", + "sample_name", + "status", + "runtime_s", + "start_memory_gb", + "end_memory_gb", + "peak_memory_gb", + "memory_change_gb", +] + + +def runtime_usage_row(input_file: str | Path, status: str, usage: FileUsage) -> dict[str, str]: + path = Path(input_file) + change_gb = "" + if usage.start_rss_bytes is not None and usage.end_rss_bytes is not None: + change_gb = f"{(usage.end_rss_bytes - usage.start_rss_bytes) / (1024 ** 3):.6f}" + return { + "raw_file": str(input_file), + "sample_name": path.stem, + "status": str(status), + "runtime_s": f"{usage.elapsed_seconds:.3f}", + "start_memory_gb": bytes_to_gb(usage.start_rss_bytes), + "end_memory_gb": bytes_to_gb(usage.end_rss_bytes), + "peak_memory_gb": bytes_to_gb(usage.peak_rss_bytes), + "memory_change_gb": change_gb, + } + + +def append_runtime_usage_tsv( + output_path: str | Path, + input_file: str | Path, + status: str, + usage: FileUsage, +) -> Path: + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + write_header = not output_path.exists() or output_path.stat().st_size == 0 + with output_path.open("a", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=RUNTIME_TSV_HEADERS, delimiter="\t") + if write_header: + writer.writeheader() + writer.writerow(runtime_usage_row(input_file, status, usage)) + return output_path diff --git a/workflow/README.md b/workflow/README.md index 3700728..a78ecdd 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -1,56 +1,161 @@ -# MetaXtract Workflow +# MetaXtract Snakemake Workflow ## Overview -This workflow automates downloading RAW files from the PRIDE -archive and analyzing them with MetaXtract. +This workflow provides a portable batch-processing layer for MetaXtract. It can +run local RAW files, including the bundled `../data/small.RAW` example, or +optionally download RAW files from the PRIDE FTP archive before analysis. +The workflow calls the same MetaXtract command-line interface used by Docker and +Apptainer/Singularity. -## Features +## Execution Modes -- Automatically downloads `.raw` files from the PRIDE FTP archive\ -- Selects newest N files per month\ -- Runs MetaXtract on every downloaded file\ -- Stores MetaXtract `_info_*.txt` logs\ +The workflow is controlled by `workflow/config.yaml`. -## Installation +```yaml +execution: + mode: "local" # local, docker, apptainer, or singularity + metaxtract_root: ".." + docker_image: "metaxtract:latest" + apptainer_image: "../metaxtract.sif" +``` + +Available modes: + +- `local`: run `../main.py --config ...` directly. +- `docker`: run the `metaxtract:latest` Docker image. +- `apptainer` or `singularity`: run the `metaxtract.sif` container image. + +For Linux container execution, keep `ms_method` and `lc_method` disabled because +Thermo method extraction is not supported by the Linux RawFileReader runtime. + +## Install Snakemake + +Create a workflow environment: + +```bash +cd workflow +python -m venv .snakemake-env +source .snakemake-env/bin/activate +pip install -r requirements.txt +``` + +On WSL2 Ubuntu, the same commands can be used from the WSL2 terminal. For best +Snakemake filesystem behavior, keep the repository under the WSL2 Linux +filesystem, for example under `~/`, rather than under a Windows-mounted path +such as `/mnt/c` or `/mnt/d`. + +## Run the Bundled Small Example + +The default `config.yaml` runs the bundled RAW file: + +```yaml +inputs: + mode: "local" + local_files: + - "../data/small.RAW" +``` + +Run a dry run first: + +```bash +snakemake -n -p --cores 1 +``` + +Run the workflow: + +```bash +snakemake -p --cores 1 +``` + +Outputs are written under: + +```text +workflow/results/metaxtract/ +workflow/results/runtime/ +``` -Create and activate a dedicated environment: +## Run with Docker - `python -m venv .snakemake-env` +Build the Docker image from the repository root: -Activate it: +```bash +cd .. +docker build -t metaxtract:latest . +cd workflow +``` - `.\.snakemake-env\Scripts\activate` +Set `execution.mode` in `config.yaml`: + +```yaml +execution: + mode: "docker" +``` -Install dependencies: +Run: + +```bash +snakemake -p --cores 1 +``` - `pip install "pulp==2.7.0"` +## Run with Apptainer/Singularity -Snakemake will run using this environment. -Or: -`pip install -U snakemake` -`pip install -U "pulp>=2.8"` +Build the image from the repository root: -## Useful Debugging ```bash -snakemake -j 1 # real run -snakemake -n # dry-run (shows what would run) -snakemake -p -j 1 # prints executed commands and logs -snakemake --forceall -j 1 # re-run everything even if outputs exist +cd .. +apptainer build metaxtract.sif apptainer.def +cd workflow ``` -## Configuration -All parameters are defined in `config.yaml`. +Set `execution.mode` in `config.yaml`: + +```yaml +execution: + mode: "apptainer" + apptainer_image: "../metaxtract.sif" +``` + +Run: + +```bash +snakemake -p --cores 1 +``` -Example: -```sh - pride: - url: "ftp://ftp.pride.ebi.ac.uk/pride/data/archive" - year: 2025 - month: 1 - max_files: 2 - copy_dir: "results/data" +On HPC systems, submit the Snakemake command through the scheduler according to +the cluster policy, for example inside a SLURM job script. - output_dir: "results" +## Optional PRIDE Download Mode + +To download RAW files from PRIDE before analysis, switch the input mode: + +```yaml +inputs: + mode: "pride" + +pride: + url: "ftp://ftp.pride.ebi.ac.uk/pride/data/archive" + year: 2025 + month: 1 + max_files: 2 + copy_dir: "results/data" +``` + +Then run: + +```bash +snakemake -p --cores 1 +``` + +Many HPC systems restrict compute-node internet access. In that case, use +`inputs.mode: "local"` and point `inputs.local_files` to RAW files that have +already been copied to the cluster. + +## Useful Commands + +```bash +snakemake -n -p --cores 1 # dry run +snakemake -p --cores 1 # run workflow +snakemake --forceall -p --cores 1 # rerun all steps ``` diff --git a/workflow/Snakefile b/workflow/Snakefile index 0996a11..7b3d1c4 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,237 +1,120 @@ configfile: "config.yaml" from pathlib import Path -from urllib.parse import urlparse -import json, time, csv, statistics +import json -OUT_DIR = Path(config["output_dir"]) -COPY_DIR = Path(config["pride"]["copy_dir"]) -PRIDE_URL = str(config["pride"]["url"]).rstrip("/") -PRIDE_YEAR = int(config["pride"]["year"]) -PRIDE_MONTH = str(config["pride"].get("month", 1)).zfill(2) -MAX_FILES = int(config["pride"]["max_files"]) -OUT_DIR_STR = str(OUT_DIR).replace("\\", "/") -COPY_DIR_STR = str(COPY_DIR).replace("\\", "/") +try: + WORKFLOW_DIR = Path(workflow.basedir).resolve() +except Exception: + WORKFLOW_DIR = Path.cwd().resolve() -def _samples_from_manifest(manifest_path: Path): - names = set() + +def _resolve_path(value): + path = Path(str(value)).expanduser() + if path.is_absolute(): + return path + cwd_path = (Path.cwd() / path).resolve() + if cwd_path.exists(): + return cwd_path + return (WORKFLOW_DIR / path).resolve() + + +def _unix(path): + return str(path).replace("\\", "/") + + +OUT_DIR = _resolve_path(config.get("output_dir", "results")) +OUT_DIR_STR = _unix(OUT_DIR) + + +def _samples_from_manifest(manifest_path): try: - data = json.loads(manifest_path.read_text(encoding="utf-8")) or {} - for rec in data.get("copied", []): - if rec.get("sample"): - names.add(str(rec["sample"])) - elif rec.get("dst"): - names.add(Path(rec["dst"]).stem) - elif rec.get("path"): - names.add(Path(rec["path"]).stem) + data = json.loads(Path(manifest_path).read_text(encoding="utf-8")) or {} except Exception: - pass - for p in COPY_DIR.glob("*.raw"): - names.add(p.stem) - return sorted(names) + return [] + return sorted({str(rec["sample"]) for rec in data.get("files", []) if rec.get("sample")}) -def out_info(sample: str) -> str: - return f"{OUT_DIR_STR}/metaxtract/{sample}/{sample}_info.tsv" -def out_ms(sample: str) -> str: - return f"{OUT_DIR_STR}/metaxtract/{sample}/{sample}_MS_method.txt" +def _raw_from_manifest(manifest_path, sample): + data = json.loads(Path(manifest_path).read_text(encoding="utf-8")) or {} + for rec in data.get("files", []): + if str(rec.get("sample")) == str(sample): + return rec["path"] + raise ValueError(f"No RAW file found in manifest for sample '{sample}'.") -def out_lc(sample: str) -> str: - return f"{OUT_DIR_STR}/metaxtract/{sample}/{sample}_LC_method.txt" rule all: input: f"{OUT_DIR_STR}/runtime/runtime_summary.txt" + rule _start_clock: output: f"{OUT_DIR_STR}/runtime/_start.timestamp" run: - Path(f"{OUT_DIR_STR}/runtime").mkdir(parents=True, exist_ok=True) + import time + Path(output[0]).parent.mkdir(parents=True, exist_ok=True) Path(output[0]).write_text(str(int(time.time())), encoding="utf-8") -checkpoint fetch_pride_raws: + +checkpoint prepare_inputs: input: f"{OUT_DIR_STR}/runtime/_start.timestamp" output: manifest=f"{OUT_DIR_STR}/runtime/manifest.json", - flag=f"{OUT_DIR_STR}/runtime/_fetch_done.flag" + flag=f"{OUT_DIR_STR}/runtime/_inputs_ready.flag" benchmark: - f"{OUT_DIR_STR}/runtime/benchmark_download.tsv" - run: - from ftplib import FTP - - COPY_DIR.mkdir(parents=True, exist_ok=True) - url = urlparse(PRIDE_URL) - host, base = url.hostname, url.path - month_dir = f"{base}/{PRIDE_YEAR}/{PRIDE_MONTH}" - - files = [] - try: - with FTP(host, timeout=30) as ftp: - ftp.login() - try: - ftp.voidcmd("OPTS MLST type;size;modify;") - entries = list(ftp.mlsd(month_dir)) - except Exception: - entries = [] - ftp.cwd(month_dir) - tmp = [] - ftp.retrlines("LIST", tmp.append) - for line in tmp: - parts = line.split(maxsplit=8) - if len(parts) < 9: - continue - name = parts[8] - kind = "dir" if line.startswith("d") else "file" - entries.append((name, {"type": kind})) - - project_dirs = [name for name, meta in entries if meta.get("type") == "dir"] - - for proj in project_dirs: - proj_path = f"{month_dir}/{proj}" - try: - try: - ftp.voidcmd("OPTS MLST type;size;modify;") - sub = list(ftp.mlsd(proj_path)) - except Exception: - sub = [] - ftp.cwd(proj_path) - tmp = [] - ftp.retrlines("LIST", tmp.append) - for line in tmp: - parts = line.split(maxsplit=8) - if len(parts) < 9: - continue - name = parts[8] - kind = "dir" if line.startswith("d") else "file" - size = parts[4] if len(parts) > 4 else "0" - sub.append((name, {"type": kind, "size": size})) - for name, meta in sub: - if name.lower().endswith(".raw"): - files.append( - { - "path": f"{proj_path}/{name}", - "size": int(meta.get("size", "0") or 0), - "sample": Path(name).stem, - } - ) - except Exception: - pass - except Exception: - files = [] - - files.sort(key=lambda x: x["path"], reverse=True) - pick = files if MAX_FILES == 0 else files[:MAX_FILES] - - copied = [] - try: - with FTP(host, timeout=60) as ftp: - ftp.login() - for rec in pick: - dst = COPY_DIR / Path(rec["path"]).name - if not dst.exists() or (rec["size"] and dst.stat().st_size != rec["size"]): - print("Downloading", rec["path"]) - with open(dst, "wb") as fh: - ftp.retrbinary(f"RETR {rec['path']}", fh.write) - copied.append({"src": rec["path"], "dst": str(dst), "sample": rec["sample"]}) - except Exception: - copied = [] - - manifest = { - "requested": MAX_FILES, - "available": len(files), - "copied_count": len(copied), - "copied": copied, - "year": PRIDE_YEAR, - "month": int(PRIDE_MONTH), - } - Path(output.manifest).write_text(json.dumps(manifest, indent=2), encoding="utf-8") - Path(output.flag).write_text("ok", encoding="utf-8") - -def analysis_targets(wc): - ck = checkpoints.fetch_pride_raws.get(**wc) - manifest_path = Path(ck.output.manifest) - samples = _samples_from_manifest(manifest_path) - targets = [] - for s in samples: - targets.append(out_info(s)) - targets.append(out_ms(s)) - targets.append(out_lc(s)) - return targets + f"{OUT_DIR_STR}/runtime/benchmark_prepare_inputs.tsv" + params: + workflow_dir=str(WORKFLOW_DIR) + script: + "scripts/prepare_inputs.py" -rule analyze_one: - input: - raw=lambda wc: f"{COPY_DIR_STR}/{wc.sample}.raw", - manifest=f"{OUT_DIR_STR}/runtime/manifest.json", - done=f"{OUT_DIR_STR}/runtime/_fetch_done.flag" - output: - info=out_info("{sample}"), - ms=out_ms("{sample}"), - lc=out_lc("{sample}") - benchmark: - f"{OUT_DIR_STR}/runtime/benchmark_analyze_{{sample}}.tsv" - run: - from pathlib import Path - from raw_parser import MetaXtract - from cli_parser import write_info_tsv, remove_empty_lines - sample = wildcards.sample - outdir = OUT_DIR / "metaxtract" / sample - outdir.mkdir(parents=True, exist_ok=True) +def _manifest_input(wildcards): + ck = checkpoints.prepare_inputs.get(**wildcards) + return ck.output.manifest + - raw = MetaXtract(input.raw) +def _raw_input(wildcards): + ck = checkpoints.prepare_inputs.get(**wildcards) + return _raw_from_manifest(ck.output.manifest, wildcards.sample) - write_info_tsv(raw, output.info) - with open(output.ms, "w", encoding="utf-8", errors="replace") as f: - f.write(raw.GetMSMethod() or "") - remove_empty_lines(output.ms) +def _analysis_targets(wildcards): + ck = checkpoints.prepare_inputs.get(**wildcards) + return [ + f"{OUT_DIR_STR}/metaxtract/{sample}/.metaxtract_done" + for sample in _samples_from_manifest(ck.output.manifest) + ] - with open(output.lc, "w", encoding="utf-8", errors="replace") as f: - f.write(raw.GetLCMethod() or "") - remove_empty_lines(output.lc) - raw.CloseRAWFile() +rule analyze_one: + input: + raw=_raw_input, + manifest=_manifest_input, + ready=f"{OUT_DIR_STR}/runtime/_inputs_ready.flag" + output: + done=f"{OUT_DIR_STR}/metaxtract/{{sample}}/.metaxtract_done", + config=f"{OUT_DIR_STR}/runtime/configs/{{sample}}.yml" + benchmark: + f"{OUT_DIR_STR}/runtime/benchmark_analyze_{{sample}}.tsv" + params: + workflow_dir=str(WORKFLOW_DIR), + output_root=f"{OUT_DIR_STR}/metaxtract" + script: + "scripts/run_metaxtract.py" - try: - Path(input.raw).unlink() - except Exception: - pass rule summarize_runtime: input: start=f"{OUT_DIR_STR}/runtime/_start.timestamp", - fetch_bench=f"{OUT_DIR_STR}/runtime/benchmark_download.tsv", - analyzed=analysis_targets, - analysis_benches=lambda wc: [str(p).replace("\\", "/") for p in (OUT_DIR / "runtime").glob("benchmark_analyze_*.tsv")] + manifest=_manifest_input, + analyzed=_analysis_targets output: f"{OUT_DIR_STR}/runtime/runtime_summary.txt" - run: - start_ts = int(Path(input.start).read_text().strip()) if Path(input.start).exists() else int(time.time()) - overall = int(time.time()) - start_ts - - def read_wall(tsv): - try: - with open(tsv, newline="") as f: - rdr = csv.DictReader(f, delimiter="\t") - row = next(rdr) - return float(row.get("wallclock", 0.0)) - except Exception: - return 0.0 - - dl = read_wall(input.fetch_bench) - walls = [read_wall(p) for p in input.analysis_benches] - total = sum(walls) - mean = statistics.mean(walls) if walls else 0.0 - - outp = Path(output[0]) - outp.parent.mkdir(parents=True, exist_ok=True) - with open(outp, "w", encoding="utf-8") as f: - f.write("=== Runtime Summary ===\n") - f.write(f"overall_pipeline: {overall}\n") - f.write(f"download_only: {dl:.3f}\n") - f.write(f"analysis_total: {total:.3f}\n") - f.write(f"analysis_per_file_mean: {mean:.3f}\n") - f.write(f"analysis_files_count: {len(walls)}\n") + params: + runtime_dir=f"{OUT_DIR_STR}/runtime" + script: + "scripts/summarize_runtime.py" diff --git a/workflow/config.yaml b/workflow/config.yaml index 6d7f0b2..0b5e946 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -1,3 +1,21 @@ +output_dir: "results" + +execution: + # local: run ../main.py directly + # docker: run the metaxtract Docker image + # apptainer/singularity: run the metaxtract .sif image + mode: "local" + metaxtract_root: ".." + docker_image: "metaxtract:latest" + apptainer_image: "../metaxtract.sif" + +inputs: + # local: use files listed below + # pride: download RAW files from PRIDE using the pride section + mode: "local" + local_files: + - "../data/small.RAW" + pride: url: "ftp://ftp.pride.ebi.ac.uk/pride/data/archive" year: 2025 @@ -5,5 +23,42 @@ pride: max_files: 2 copy_dir: "results/data" -output_dir: "results" +metaxtract: + outputs: + file_based_details: true + ms_method: false + lc_method: false + ms2_peaklist_export: true + ms1_peaklist_export: true + ms2_technical_details_export: false + ms1_technical_details_export: false + hdf5_export: false + + scan_header: + MS1: + select_all: true + columns: + Ion Injection Time (ms): true + Total Number of Peaks: true + Total Ion Current: true + Scan Start Time (min): true + Base Peak Intensity: true + Base Peak m/z: true + Scan Mode: true + thermo_Multi Inject Info: true + thermo_Multiple Injection: true + MS2: + select_all: true + columns: + Total Ion Current: true + Total Number of Peaks: true + Scan Start Time (min): true + Base Peak Intensity: true + Base Peak m/z: true + Selected Ion Intensity: true + Filter String: true + Scan Mode: true + visualisation: + enabled: true + format: html diff --git a/workflow/requirements.txt b/workflow/requirements.txt new file mode 100644 index 0000000..12b47c8 --- /dev/null +++ b/workflow/requirements.txt @@ -0,0 +1 @@ +snakemake>=8 diff --git a/workflow/scripts/prepare_inputs.py b/workflow/scripts/prepare_inputs.py new file mode 100644 index 0000000..27da945 --- /dev/null +++ b/workflow/scripts/prepare_inputs.py @@ -0,0 +1,132 @@ +from pathlib import Path +from urllib.parse import urlparse +import json + + +def resolve_path(value, workflow_dir): + path = Path(str(value)).expanduser() + if path.is_absolute(): + return path + cwd_path = (Path.cwd() / path).resolve() + if cwd_path.exists(): + return cwd_path + return (Path(workflow_dir) / path).resolve() + + +def unix(path): + return str(path).replace("\\", "/") + + +cfg = snakemake.config +workflow_dir = Path(snakemake.params.workflow_dir) +input_cfg = cfg.get("inputs", {}) or {} +input_mode = str(input_cfg.get("mode", "local")).lower() +records = [] + +if input_mode == "local": + local_files = [resolve_path(path, workflow_dir) for path in input_cfg.get("local_files", [])] + if not local_files: + raise ValueError("inputs.local_files must contain at least one RAW file when inputs.mode is local.") + + seen = set() + for raw_path in local_files: + if not raw_path.exists(): + raise FileNotFoundError(f"RAW file not found: {raw_path}") + sample = raw_path.stem + if sample in seen: + raise ValueError(f"Duplicate sample name in local inputs: {sample}") + seen.add(sample) + records.append({"sample": sample, "path": unix(raw_path), "source": "local"}) + +elif input_mode == "pride": + from ftplib import FTP + + pride_cfg = cfg.get("pride", {}) or {} + pride_url = str(pride_cfg.get("url", "ftp://ftp.pride.ebi.ac.uk/pride/data/archive")).rstrip("/") + pride_year = int(pride_cfg.get("year", 2025)) + pride_month = str(pride_cfg.get("month", 1)).zfill(2) + max_files = int(pride_cfg.get("max_files", 2)) + copy_dir = resolve_path(pride_cfg.get("copy_dir", "results/data"), workflow_dir) + copy_dir.mkdir(parents=True, exist_ok=True) + + url = urlparse(pride_url) + host, base = url.hostname, url.path + month_dir = f"{base}/{pride_year}/{pride_month}" + files = [] + + with FTP(host, timeout=30) as ftp: + ftp.login() + try: + ftp.voidcmd("OPTS MLST type;size;modify;") + entries = list(ftp.mlsd(month_dir)) + except Exception: + entries = [] + ftp.cwd(month_dir) + tmp = [] + ftp.retrlines("LIST", tmp.append) + for line in tmp: + parts = line.split(maxsplit=8) + if len(parts) < 9: + continue + kind = "dir" if line.startswith("d") else "file" + entries.append((parts[8], {"type": kind})) + + project_dirs = [name for name, meta in entries if meta.get("type") == "dir"] + for project in project_dirs: + project_path = f"{month_dir}/{project}" + try: + try: + ftp.voidcmd("OPTS MLST type;size;modify;") + subentries = list(ftp.mlsd(project_path)) + except Exception: + subentries = [] + ftp.cwd(project_path) + tmp = [] + ftp.retrlines("LIST", tmp.append) + for line in tmp: + parts = line.split(maxsplit=8) + if len(parts) < 9: + continue + kind = "dir" if line.startswith("d") else "file" + size = parts[4] if len(parts) > 4 else "0" + subentries.append((parts[8], {"type": kind, "size": size})) + + for name, meta in subentries: + if name.lower().endswith(".raw"): + files.append( + { + "path": f"{project_path}/{name}", + "size": int(meta.get("size", "0") or 0), + "sample": Path(name).stem, + } + ) + except Exception: + pass + + files.sort(key=lambda item: item["path"], reverse=True) + selected = files if max_files == 0 else files[:max_files] + + with FTP(host, timeout=60) as ftp: + ftp.login() + for rec in selected: + dst = copy_dir / Path(rec["path"]).name + if not dst.exists() or (rec["size"] and dst.stat().st_size != rec["size"]): + print("Downloading", rec["path"]) + with open(dst, "wb") as handle: + ftp.retrbinary(f"RETR {rec['path']}", handle.write) + records.append({"sample": rec["sample"], "path": unix(dst.resolve()), "source": rec["path"]}) + +else: + raise ValueError("inputs.mode must be local or pride.") + +execution_mode = str((cfg.get("execution", {}) or {}).get("mode", "local")).lower() +manifest = { + "input_mode": input_mode, + "execution_mode": execution_mode, + "file_count": len(records), + "files": records, +} + +Path(snakemake.output.manifest).parent.mkdir(parents=True, exist_ok=True) +Path(snakemake.output.manifest).write_text(json.dumps(manifest, indent=2), encoding="utf-8") +Path(snakemake.output.flag).write_text("ok", encoding="utf-8") diff --git a/workflow/scripts/run_metaxtract.py b/workflow/scripts/run_metaxtract.py new file mode 100644 index 0000000..4341bd5 --- /dev/null +++ b/workflow/scripts/run_metaxtract.py @@ -0,0 +1,99 @@ +from pathlib import Path +import json +import subprocess +import sys + + +def resolve_path(value, workflow_dir): + path = Path(str(value)).expanduser() + if path.is_absolute(): + return path + cwd_path = (Path.cwd() / path).resolve() + if cwd_path.exists(): + return cwd_path + return (Path(workflow_dir) / path).resolve() + + +def unix(path): + return str(path).replace("\\", "/") + + +def metaxtract_config(raw_path, output_root, meta_cfg, container=False): + raw_path = Path(raw_path) + if container: + raw_input = f"/data/{raw_path.name}" + output_dir = "/out" + else: + raw_input = unix(raw_path.resolve()) + output_dir = unix(Path(output_root).resolve()) + + return { + "io": { + "input": [raw_input], + "output_dir": output_dir, + }, + "outputs": meta_cfg.get("outputs", {}), + "scan_header": meta_cfg.get("scan_header", {}), + "visualisation": meta_cfg.get("visualisation", {"enabled": False, "format": "html"}), + "multi_comparison": {"enabled": False}, + } + + +cfg = snakemake.config +workflow_dir = Path(snakemake.params.workflow_dir) +exec_cfg = cfg.get("execution", {}) or {} +exec_mode = str(exec_cfg.get("mode", "local")).lower() +meta_cfg = cfg.get("metaxtract", {}) or {} + +raw_path = Path(snakemake.input.raw).resolve() +output_root = Path(snakemake.params.output_root).resolve() +config_path = Path(snakemake.output.config).resolve() +output_root.mkdir(parents=True, exist_ok=True) +config_path.parent.mkdir(parents=True, exist_ok=True) + +if exec_mode == "local": + run_cfg = metaxtract_config(raw_path, output_root, meta_cfg, container=False) + config_path.write_text(json.dumps(run_cfg, indent=2), encoding="utf-8") + metaxtract_root = resolve_path(exec_cfg.get("metaxtract_root", ".."), workflow_dir) + cmd = [sys.executable, str(metaxtract_root / "main.py"), "--config", str(config_path)] + +elif exec_mode == "docker": + run_cfg = metaxtract_config(raw_path, output_root, meta_cfg, container=True) + config_path.write_text(json.dumps(run_cfg, indent=2), encoding="utf-8") + docker_image = str(exec_cfg.get("docker_image", "metaxtract:latest")) + cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{raw_path.parent}:/data:ro", + "-v", + f"{output_root}:/out", + "-v", + f"{config_path}:/config.yml:ro", + docker_image, + "--config", + "/config.yml", + ] + +elif exec_mode in {"apptainer", "singularity"}: + run_cfg = metaxtract_config(raw_path, output_root, meta_cfg, container=True) + config_path.write_text(json.dumps(run_cfg, indent=2), encoding="utf-8") + runner = "apptainer" if exec_mode == "apptainer" else "singularity" + apptainer_image = resolve_path(exec_cfg.get("apptainer_image", "../metaxtract.sif"), workflow_dir) + cmd = [ + runner, + "run", + "--bind", + f"{raw_path.parent}:/data:ro,{output_root}:/out,{config_path}:/config.yml:ro", + str(apptainer_image), + "--config", + "/config.yml", + ] + +else: + raise ValueError("execution.mode must be local, docker, apptainer, or singularity.") + +subprocess.run(cmd, check=True) +Path(snakemake.output.done).parent.mkdir(parents=True, exist_ok=True) +Path(snakemake.output.done).write_text("ok", encoding="utf-8") diff --git a/workflow/scripts/summarize_runtime.py b/workflow/scripts/summarize_runtime.py new file mode 100644 index 0000000..40b5081 --- /dev/null +++ b/workflow/scripts/summarize_runtime.py @@ -0,0 +1,39 @@ +from pathlib import Path +import csv +import json +import statistics +import time + + +def read_wall(tsv): + try: + with open(tsv, newline="") as handle: + row = next(csv.DictReader(handle, delimiter="\t")) + return float(row.get("wallclock") or row.get("s") or 0.0) + except Exception: + return 0.0 + + +runtime_dir = Path(snakemake.params.runtime_dir) +start_path = Path(snakemake.input.start) +start_ts = int(start_path.read_text().strip()) if start_path.exists() else int(time.time()) +overall = int(time.time()) - start_ts + +prepare_wall = read_wall(runtime_dir / "benchmark_prepare_inputs.tsv") +analysis_benches = sorted(runtime_dir.glob("benchmark_analyze_*.tsv")) +analysis_walls = [read_wall(path) for path in analysis_benches] +total = sum(analysis_walls) +mean = statistics.mean(analysis_walls) if analysis_walls else 0.0 + +manifest = json.loads(Path(snakemake.input.manifest).read_text(encoding="utf-8")) +outp = Path(snakemake.output[0]) +outp.parent.mkdir(parents=True, exist_ok=True) +with open(outp, "w", encoding="utf-8") as handle: + handle.write("=== Runtime Summary ===\n") + handle.write(f"input_mode: {manifest.get('input_mode')}\n") + handle.write(f"execution_mode: {manifest.get('execution_mode')}\n") + handle.write(f"files_count: {manifest.get('file_count', 0)}\n") + handle.write(f"overall_pipeline_seconds: {overall}\n") + handle.write(f"prepare_inputs_seconds: {prepare_wall:.3f}\n") + handle.write(f"analysis_total_seconds: {total:.3f}\n") + handle.write(f"analysis_per_file_mean_seconds: {mean:.3f}\n") From 95317e6f923ae00872dac0cc0fc95a7fb24710d0 Mon Sep 17 00:00:00 2001 From: Ahmad lutfi Date: Fri, 28 Aug 2026 16:51:25 +0200 Subject: [PATCH 6/8] pride --- pride_downloads.sh | 237 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 pride_downloads.sh diff --git a/pride_downloads.sh b/pride_downloads.sh new file mode 100644 index 0000000..ae02df8 --- /dev/null +++ b/pride_downloads.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +set -euo pipefail + +BASE_DIR="MetaXtract_Thermo_RAW_Showcase" +MANIFEST="${BASE_DIR}/raw_file_manifest.tsv" + +mkdir -p "${BASE_DIR}" + +download_raw() { + local instrument="$1" + local pxd="$2" + local acquisition="$3" + local url="$4" + + local filename + local folder + local output + + filename="$(basename "${url}")" + folder="${BASE_DIR}/${instrument}_${pxd}" + output="${folder}/${filename}" + + mkdir -p "${folder}" + + echo + echo "============================================================" + echo "Instrument : ${instrument}" + echo "PXD : ${pxd}" + echo "Acquisition: ${acquisition}" + echo "File : ${filename}" + echo "============================================================" + + wget \ + --continue \ + --show-progress \ + --output-document="${output}" \ + "${url}" +} + + +# ============================================================ +# Orbitrap Astral +# PXD072131 +# DIA +# ============================================================ + +download_raw \ + "Orbitrap_Astral" \ + "PXD072131" \ + "DIA" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/07/PXD072131/AST_SJE_20241213_E24_003_8_15cm.raw" + +download_raw \ + "Orbitrap_Astral" \ + "PXD072131" \ + "DIA" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/07/PXD072131/AST_SJE_20241213_E24_003_7.raw" + + +# ============================================================ +# Orbitrap Ascend +# PXD070486 +# +# Top-down / targeted MS/MS +# File name explicitly indicates targeted ETD. +# ============================================================ + +download_raw \ + "Orbitrap_Ascend" \ + "PXD070486" \ + "Targeted_ETD" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/02/PXD070486/20240326_SPM_50nl_25ng_OT_7p5_120k_res_Targeted_ETD_3ms_10.raw" + + +# ============================================================ +# Orbitrap Exploris 480 +# PXD069212 +# DIA +# ============================================================ + +download_raw \ + "Orbitrap_Exploris_480" \ + "PXD069212" \ + "DIA" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/04/PXD069212/01_TK_ENplus_Hu_Plasma_DIA_Source-1_1.raw" + + +# ============================================================ +# Orbitrap Fusion Lumos +# PXD081636 +# +# TMTpro MS3 experiment. DDA +# This is essentially a DDA/TMT-MS3 workflow rather than DIA. +# ============================================================ + +download_raw \ + "Orbitrap_Fusion_Lumos" \ + "PXD081636" \ + "DDA_TMTpro_MS3" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/07/PXD081636/YeastAgingvsWhi5atc_Std_F10_R1_T1_TMTMS3pro001_002522_JASK013_FL1.raw" + + +# ============================================================ +# PXD083028 +# Q Exactive HF +# DDA library fraction. +# +# The filename explicitly contains DDALib. +# I keep the instrument folder conservative here until the +# ============================================================ + +download_raw \ + "Q_Exactive_HF" \ + "PXD083028" \ + "DDA_Library" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/08/PXD083028/20220106_QE5_YABH_RDS3363_PlasmaProt_Mouse_GDF15_DDALib_NonDe_F02.raw" + + +# ============================================================ +# Orbitrap Eclipse +# PXD082542 +# Proteomics experiment +# ============================================================ + +download_raw \ + "Orbitrap_Eclipse" \ + "PXD082542" \ + "DDA" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/08/PXD082542/20250417_Khavari_IFerguson_15015_5_C1.raw" + + +# ============================================================ +# PXD080728 +# +# Orbitrap Astral Zoom +# DIA +# ============================================================ + +download_raw \ + "Orbitrap_Astral_Zoom" \ + "PXD080728" \ + "DIA" \ + "https://ftp.pride.ebi.ac.uk/pride/data/archive/2026/08/PXD080728/20260629_AZ2_NEO6_SCL-IAH_50ID_HeLa_250pg_425-625_10.raw" + + + +printf "Instrument\tPXD\tAcquisition\tFilename\tSize_bytes\tsize_gb\n" \ + > "${MANIFEST}" + +add_to_manifest() { + local instrument="$1" + local pxd="$2" + local acquisition="$3" + local filename="$4" + + local path="${BASE_DIR}/${instrument}_${pxd}/${filename}" + + if [[ ! -f "${path}" ]]; then + echo "WARNING: file not found: ${path}" >&2 + return + fi + + local size_bytes + local size_gb + + size_bytes="$(stat -c '%s' "${path}")" + size_gb="$(du -h "${path}" | cut -f1)" + + printf "%s\t%s\t%s\t%s\t%s\t%s\n" \ + "${instrument}" \ + "${pxd}" \ + "${acquisition}" \ + "${filename}" \ + "${size_bytes}" \ + "${size_gb}" \ + >> "${MANIFEST}" +} + + +add_to_manifest \ + "Orbitrap_Astral" \ + "PXD072131" \ + "DIA" \ + "AST_SJE_20241213_E24_003_8_15cm.raw" + +add_to_manifest \ + "Orbitrap_Astral" \ + "PXD072131" \ + "DIA" \ + "AST_SJE_20241213_E24_003_7.raw" + +add_to_manifest \ + "Orbitrap_Ascend" \ + "PXD070486" \ + "Targeted_ETD" \ + "20240326_SPM_50nl_25ng_OT_7p5_120k_res_Targeted_ETD_3ms_10.raw" + +add_to_manifest \ + "Orbitrap_Exploris_480" \ + "PXD069212" \ + "DIA" \ + "01_TK_ENplus_Hu_Plasma_DIA_Source-1_1.raw" + +add_to_manifest \ + "Orbitrap_Fusion_Lumos" \ + "PXD081636" \ + "DDA_TMTpro_MS3" \ + "YeastAgingvsWhi5atc_Std_F10_R1_T1_TMTMS3pro001_002522_JASK013_FL1.raw" + +add_to_manifest \ + "Q_Exactive_HF" \ + "PXD083028" \ + "DDA_Library" \ + "20220106_QE5_YABH_RDS3363_PlasmaProt_Mouse_GDF15_DDALib_NonDe_F02.raw" + +add_to_manifest \ + "Orbitrap_Eclipse" \ + "PXD082542" \ + "DDA" \ + "20250417_Khavari_IFerguson_15015_5_C1.raw" + +add_to_manifest \ + "Orbitrap_Astral_Zoom" \ + "PXD080728" \ + "DIA" \ + "20260629_AZ2_NEO6_SCL-IAH_50ID_HeLa_250pg_425-625_10.raw" + + +echo +echo "============================================================" +echo "Downloads complete." +echo "Manifest: ${MANIFEST}" +echo "============================================================" +echo + +column -t -s $'\t' "${MANIFEST}" || cat "${MANIFEST}" \ No newline at end of file From 6ba75738acce2f44d6192436627a406a2ae74e67 Mon Sep 17 00:00:00 2001 From: Ahmad lutfi Date: Fri, 28 Aug 2026 17:48:10 +0200 Subject: [PATCH 7/8] sdrf_draft --- README.md | 64 ++++++++++- cli_parser.py | 103 +++++++++++++++++ config.yml | 10 ++ main.py | 21 +++- sdrf_export.py | 236 +++++++++++++++++++++++++++++++++++++- tests/test_sdrf_export.py | 91 +++++++++++++++ 6 files changed, 522 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 802cd37..418f3e8 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,16 @@ visualisation: enabled: true format: html +sdrf: + # Optional CLI/HPC SDRF support. + # draft: true writes metadata.sdrf.draft.tsv during the normal run, with + # only RAW-derived fields filled. Users can complete it after processing. + draft: false + draft_output: metadata.sdrf.draft.tsv + # metadata can point to a completed user-filled SDRF metadata TSV. + metadata: null + output: metadata.sdrf.tsv + multi_comparison: enabled: true # Select any 2 or more inputs using their 1-based positions in io.input. @@ -396,12 +406,64 @@ Interactive Plotly HTML reports, MS1 and MS2 trends, and cross-sample overlays a For every processed RAW file, both the GUI log and CLI output report memory at the start and a final summary containing runtime, ending memory, sampled peak memory, and memory change. Each run also writes `runtime_summary_YYYYMMDD_HHMMSS.tsv` in the root output directory. The TSV contains one row per processed RAW file with status, runtime in seconds, start/end/peak memory in GB, and memory change in GB. Memory is the resident set size (RSS) of the MetaXtract process, so it includes Python, native libraries, and Thermo/.NET allocations used while that file is processed. #### SDRF-Proteomics export -Enable **Export SDRF-Proteomics metadata (.sdrf.tsv)** in the GUI to open the metadata editor before processing. MetaXtract fills the RAW filename, instrument model, acquisition date, technology type, SDRF version, and template. Initially, the grid shows only required MS-proteomics inputs that cannot be determined reliably from a RAW file. +Enable **Export SDRF-Proteomics metadata (.sdrf.tsv)** in the GUI to open the metadata editor before processing. MetaXtract fills the RAW filename, instrument model, acquisition date, technology type, SDRF annotation tool, SDRF version, and template. Common Thermo instrument models are written as PSI-MS controlled-vocabulary values, for example `NT=Orbitrap Fusion Lumos;AC=MS:1002732`. If the instrument cannot be mapped automatically, fill `comment[instrument]` manually with the preferred `NT=...;AC=...` value. Initially, the grid shows only required MS-proteomics inputs that cannot be determined reliably from a RAW file. Use **Add column** to search and multi-select from the complete known column-name catalog in the official SDRF templates registry. This includes sample, clinical, organism, DIA, single-cell, crosslinking, immunopeptidomics, metaproteomics, environmental, affinity-proteomics, and metabolomics fields. A custom `factor value[...]` column can also be added from the same picker. Added columns must be filled for every row and can be removed again with **Remove optional column**. The editor starts with one sample-to-file row per selected RAW file. Additional rows can be added for multiplexed experiments where multiple samples or labels share a RAW file. The dataset-level file is written as `metadata.sdrf.tsv` in the root output directory using the `ms-proteomics v1.1.0` template. +CLI users can generate an SDRF draft during the normal extraction run: + +```bash +python main.py \ + --input path/to/sample_1.RAW path/to/sample_2.RAW \ + --output-dir output/cli_run \ + --file-based-details \ + --sdrf-draft +``` + +This writes `output/cli_run/metadata.sdrf.draft.tsv`. MetaXtract fills only the fields that can be read or derived safely from the RAW file, including the RAW filename, acquisition date, technology type, recognized instrument model, SDRF annotation tool, SDRF version, and SDRF template. Biological and experimental design fields that cannot be inferred from RAW files are left for the user to complete manually. + +For the bundled small example, the draft command is: + +```bash +python main.py \ + --input data/small.RAW \ + --output-dir output/cli_small \ + --file-based-details \ + --sdrf-draft +``` + +The same draft mode can be enabled from YAML: + +```yaml +sdrf: + draft: true + draft_output: metadata.sdrf.draft.tsv +``` + +Users who already have a completed metadata TSV can ask MetaXtract to validate/enrich it and write a final SDRF file: + +```bash +python main.py \ + --input data/small.RAW \ + --output-dir output/cli_small \ + --file-based-details \ + --sdrf-metadata sdrf_input.tsv +``` + +The completed metadata file can also be passed from YAML: + +```yaml +sdrf: + metadata: sdrf_input.tsv + output: metadata.sdrf.tsv +``` + +If a user wants a blank starter TSV without processing RAW files, they can still create one with `--sdrf-template-out`; this command only writes the TSV template and exits. + +Optional SDRF fields such as `comment[precursor mass tolerance]` and `comment[fragment mass tolerance]` can be added in the GUI, or as extra columns in the CLI metadata TSV. These fields describe the mass-error tolerance intended for downstream identification/search workflows: precursor tolerance applies to intact precursor ions, while fragment tolerance applies to MS/MS fragment ions. + --- ## Using MetaXtract as a Python Library You can import `MetaXtract` directly and use it in your own Python scripts (e.g. notebooks, pipelines, custom QC tooling). diff --git a/cli_parser.py b/cli_parser.py index 82521eb..3642fa2 100644 --- a/cli_parser.py +++ b/cli_parser.py @@ -23,6 +23,14 @@ format_bytes, format_file_usage, ) +from sdrf_export import ( + draft_sdrf_row_for_file, + enrich_sdrf_rows_for_file, + read_sdrf_user_metadata, + validate_sdrf_metadata, + write_sdrf, + write_sdrf_cli_template, +) def _cancel_requested(should_stop=None) -> bool: @@ -462,6 +470,24 @@ def write_info_tsv(raw_parser, out_tsv_path: str, should_stop=None): for sec, key, val in rows: w.writerow([_tsv_safe(sec), _tsv_safe(key), _tsv_safe(val)]) + +def _raw_instrument_name(raw_parser) -> str: + instrument_details = raw_parser.GetInstrumentDetails() or {} + instrument_candidates = ( + instrument_details.get("Instrument Model"), + instrument_details.get("Instrument Name"), + raw_parser.GetInstrumentName(), + ) + return next( + ( + str(value).strip() + for value in instrument_candidates + if value and str(value).strip().casefold() not in {"unknown", "n/a", "not available"} + ), + "", + ) + + def run_cli(args): stop_event = threading.Event() file_usage_monitor = None @@ -513,13 +539,25 @@ def request_stop(signum, _frame): cfg_inputs = _cfg_get(cfg, ["io", "input"], []) or [] cfg_outdir = _cfg_get(cfg, ["io", "output_dir"], None) + cfg_sdrf = _cfg_get(cfg, ["sdrf"], {}) or {} inputs = list(getattr(args, "input", None) or cfg_inputs) outdir = getattr(args, "output_dir", None) or cfg_outdir + sdrf_draft = bool(getattr(args, "sdrf_draft", False) or cfg_sdrf.get("draft", False)) + sdrf_metadata_path = getattr(args, "sdrf_metadata", None) or cfg_sdrf.get("metadata") + sdrf_template_out = getattr(args, "sdrf_template_out", None) or cfg_sdrf.get("template_out") + sdrf_output_name = cfg_sdrf.get("output") or "metadata.sdrf.tsv" + sdrf_draft_output_name = cfg_sdrf.get("draft_output") or "metadata.sdrf.draft.tsv" if not inputs: print("[ERROR] No input files. Set io.input in config.yml or pass --input ...") sys.exit(1) + + if sdrf_template_out: + out = write_sdrf_cli_template(sdrf_template_out, inputs) + print(f"[INFO] Wrote SDRF metadata template: {out}") + return + if not outdir: print("[ERROR] No output directory. Set io.output_dir in config.yml or pass --output-dir ...") sys.exit(1) @@ -578,6 +616,31 @@ def request_stop(signum, _frame): proc_inputs = cmp_inputs if (multi_cmp and cmp_inputs) else inputs + sdrf_rows = [] + sdrf_draft_rows = [] + sdrf_extra_columns = [] + if sdrf_metadata_path: + try: + sdrf_user_rows, sdrf_extra_columns = read_sdrf_user_metadata( + sdrf_metadata_path, + proc_inputs, + ) + errors = validate_sdrf_metadata( + sdrf_user_rows, + proc_inputs, + extra_columns=sdrf_extra_columns, + ) + except Exception as e: + print(f"[ERROR] Could not read SDRF metadata TSV: {e}") + sys.exit(1) + if errors: + print("[ERROR] Invalid SDRF metadata TSV:") + for error in errors[:20]: + print(f" - {error}") + if len(errors) > 20: + print(f" - ...and {len(errors) - 20} more errors") + sys.exit(1) + all_ms1_tic, all_ms1_bpi, all_ms1_tnp = [], [], [] all_ms2_tic, all_ms2_tnp, all_ms2_prec = [], [], [] ms1_box_tic, ms1_box_bpi, ms1_box_tnp = {}, {}, {} @@ -599,6 +662,31 @@ def request_stop(signum, _frame): finish_file_usage("Failed") raise + instrument = _raw_instrument_name(raw_parser) + acquisition_date = raw_parser.GetFileCreationDate() + + if sdrf_metadata_path: + file_sdrf_rows = enrich_sdrf_rows_for_file( + sdrf_user_rows, + input_file, + instrument, + acquisition_date, + ) + if any(not row.get("instrument") for row in file_sdrf_rows): + raw_parser.CloseRAWFile() + finish_file_usage("Failed") + print( + f"[ERROR] No instrument model was found for {input_file}. " + "Add comment[instrument] in the SDRF metadata TSV." + ) + sys.exit(1) + sdrf_rows.extend(file_sdrf_rows) + + if sdrf_draft: + sdrf_draft_rows.append( + draft_sdrf_row_for_file(input_file, instrument, acquisition_date) + ) + base = os.path.splitext(os.path.basename(input_file))[0] sample_out = os.path.join(outdir, base) os.makedirs(sample_out, exist_ok=True) @@ -743,6 +831,21 @@ def request_stop(signum, _frame): finish_file_usage("Stopped") print("[INFO] Processing stopped.") + if not stop_event.is_set() and sdrf_metadata_path: + out = write_sdrf( + Path(outdir) / sdrf_output_name, + sdrf_rows, + extra_columns=sdrf_extra_columns, + ) + print(f"[INFO] SDRF-Proteomics metadata: {out}") + + if not stop_event.is_set() and sdrf_draft: + out = write_sdrf( + Path(outdir) / sdrf_draft_output_name, + sdrf_draft_rows, + ) + print(f"[INFO] Draft SDRF-Proteomics metadata: {out}") + if not stop_event.is_set() and graphical_representation and len(all_ms1_tic) >= 2: out = Path(outdir) / "MS1_compare.html" write_comparison_html_with_boxplots( diff --git a/config.yml b/config.yml index d2695ab..6ca8a31 100644 --- a/config.yml +++ b/config.yml @@ -17,6 +17,16 @@ visualisation: enabled: true export_format: "html" +sdrf: + # Optional CLI/HPC SDRF support. + # draft: true writes metadata.sdrf.draft.tsv during the normal run, with + # only RAW-derived fields filled. Users can complete it after processing. + draft: false + draft_output: "metadata.sdrf.draft.tsv" + # metadata can point to a completed user-filled SDRF metadata TSV. + metadata: null + output: "metadata.sdrf.tsv" + multi_comparison: enabled: true samples: [1, 2, 3] # Select 2 or more samples using 1-based indices into io.input diff --git a/main.py b/main.py index 07cec7b..3504463 100644 --- a/main.py +++ b/main.py @@ -20,7 +20,10 @@ def _is_cli(args: argparse.Namespace) -> bool: bool(getattr(args, "complete_ms1", False)), bool(getattr(args, "complete_ms2", False)), bool(getattr(args, "ms2_peaklist_export", False)), - bool(getattr(args, "ms1_peaklist_export", False)) + bool(getattr(args, "ms1_peaklist_export", False)), + bool(getattr(args, "sdrf_draft", False)), + bool(getattr(args, "sdrf_metadata", None)), + bool(getattr(args, "sdrf_template_out", None)), ] ) @@ -56,6 +59,22 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--ms1-peaklist-export", dest="ms1_peaklist_export", action="store_true", help="Export MS1 peak list as Parquet",) p.add_argument("--ms2-technical-details-export", dest="ms2_technical_details_export", action="store_true", help="Export MS2 technical details CSV") p.add_argument("--ms1-technical-details-export", dest="ms1_technical_details_export", action="store_true", help="Export MS1 technical details CSV") + p.add_argument( + "--sdrf-draft", + dest="sdrf_draft", + action="store_true", + help="Write a draft SDRF TSV during the CLI run with only RAW-derived fields filled", + ) + p.add_argument( + "--sdrf-metadata", + dest="sdrf_metadata", + help="Path to a user-filled SDRF metadata TSV for CLI export", + ) + p.add_argument( + "--sdrf-template-out", + dest="sdrf_template_out", + help="Write a fillable SDRF metadata TSV template for the selected inputs and exit", + ) return p diff --git a/sdrf_export.py b/sdrf_export.py index bdb3e3a..499b59a 100644 --- a/sdrf_export.py +++ b/sdrf_export.py @@ -10,6 +10,7 @@ SDRF_VERSION = "v1.1.0" SDRF_TEMPLATE = "ms-proteomics v1.1.0" +SDRF_ANNOTATION_TOOL = "MetaXtract" TECHNOLOGY_TYPE = "proteomic profiling by mass spectrometry" ACQUISITION_METHODS = { @@ -30,6 +31,65 @@ "chymotrypsin": "NT=Chymotrypsin;AC=MS:1001306", } +INSTRUMENT_MODELS = { + "Exactive": "MS:1000649", + "Exactive Plus": "MS:1002526", + "LTQ FT": "MS:1000448", + "LTQ FT Ultra": "MS:1000557", + "LTQ Orbitrap": "MS:1000449", + "LTQ Orbitrap Classic": "MS:1002835", + "LTQ Orbitrap Discovery": "MS:1000555", + "LTQ Orbitrap Velos": "MS:1001742", + "LTQ Orbitrap Velos/ETD": "MS:1003499", + "LTQ Orbitrap XL": "MS:1000556", + "LTQ Orbitrap XL ETD": "MS:1000639", + "MALDI LTQ Orbitrap": "MS:1000643", + "MALDI LTQ Orbitrap Discovery": "MS:1003497", + "MALDI LTQ Orbitrap XL": "MS:1003496", + "Orbitrap Astral": "MS:1003378", + "Orbitrap Astral Zoom": "MS:1003442", + "Orbitrap Eclipse": "MS:1003029", + "Orbitrap Elite": "MS:1001910", + "Orbitrap Exploris 120": "MS:1003095", + "Orbitrap Exploris 240": "MS:1003094", + "Orbitrap Exploris 480": "MS:1003028", + "Orbitrap Exploris GC 240": "MS:1003423", + "Orbitrap Exploris GC-MS": "MS:1002992", + "Orbitrap Fusion": "MS:1002416", + "Orbitrap Fusion ETD": "MS:1002417", + "Orbitrap Fusion Lumos": "MS:1002732", + "Orbitrap Velos Pro": "MS:1003096", + "Q Exactive": "MS:1001911", + "Q Exactive Focus": "MS:1002993", + "Q Exactive GC Orbitrap": "MS:1003395", + "Q Exactive HF": "MS:1002523", + "Q Exactive HF-X": "MS:1002877", + "Q Exactive Plus": "MS:1002634", + "Q Exactive UHMR": "MS:1003245", + "TSQ": "MS:1000750", + "TSQ 7000": "MS:1000749", + "TSQ 8000": "MS:1003503", + "TSQ 8000 Evo": "MS:1002525", + "TSQ 9000": "MS:1002876", + "TSQ Altis": "MS:1002874", + "TSQ Altis Plus": "MS:1003292", + "TSQ Certis": "MS:1003800", + "TSQ Endura": "MS:1002419", + "TSQ Quantum": "MS:1000199", + "TSQ Quantum Access": "MS:1000644", + "TSQ Quantum Access MAX": "MS:1003498", + "TSQ Quantum Ultra": "MS:1000751", + "TSQ Quantum Ultra AM": "MS:1000743", + "TSQ Quantis": "MS:1002875", + "TSQ Quantiva": "MS:1002418", + "TSQ Vantage": "MS:1001510", +} + +INSTRUMENT_MODELS_BY_KEY = { + re.sub(r"[^a-z0-9]+", "", name.casefold()): name + for name in INSTRUMENT_MODELS +} + USER_REQUIRED_FIELDS = ( ("source_name", "source name"), ("assay_name", "assay name"), @@ -63,6 +123,7 @@ "technology type", "comment[data file]", "comment[acquisition date]", + "comment[sdrf annotation tool]", "comment[sdrf version]", "comment[sdrf template]", } @@ -97,6 +158,31 @@ def normalize_cleavage_agent(value: str) -> str: return CLEAVAGE_AGENTS.get(cleaned.casefold(), cleaned) +def _instrument_key(value: str) -> str: + cleaned = _text(value).casefold() + cleaned = re.sub(r"\bthermo(?: fisher)? scientific\b", " ", cleaned) + cleaned = re.sub(r"\bmass spectrometer\b|\bspectrometer\b", " ", cleaned) + cleaned = re.sub(r"\bms\b", " ", cleaned) + cleaned = re.sub(r"\borbitrap$", " ", cleaned) + return re.sub(r"[^a-z0-9]+", "", cleaned) + + +def normalize_instrument(value: str) -> str: + cleaned = _known_text(value) + if not cleaned or cleaned.startswith("NT="): + return cleaned + + key = _instrument_key(cleaned) + exact_name = INSTRUMENT_MODELS_BY_KEY.get(key) + if exact_name: + return f"NT={exact_name};AC={INSTRUMENT_MODELS[exact_name]}" + + for name in sorted(INSTRUMENT_MODELS, key=len, reverse=True): + if re.search(rf"\b{re.escape(name.casefold())}\b", cleaned.casefold()): + return f"NT={name};AC={INSTRUMENT_MODELS[name]}" + return cleaned + + def normalize_acquisition_date(value) -> str: cleaned = _text(value) if not cleaned: @@ -148,6 +234,126 @@ def _valid_factor_header(header: str) -> bool: return bool(re.fullmatch(r"factor value\[[^\[\]\t\r\n]+\]", header)) +def _user_field_for_header(header: str) -> str | None: + normalized = _text(header).casefold() + if normalized in {"file", "raw file", "raw_file", "comment[data file]"}: + return "file" + if normalized == "source name": + return "source_name" + if normalized == "assay name": + return "assay_name" + if normalized == "characteristics[organism]": + return "organism" + if normalized == "characteristics[organism part]": + return "organism_part" + if normalized == "characteristics[biological replicate]": + return "biological_replicate" + if normalized == "comment[proteomics data acquisition method]": + return "acquisition_method" + if normalized == "comment[label]": + return "label" + if normalized == "comment[cleavage agent details]": + return "cleavage_agent" + if normalized == "comment[fraction identifier]": + return "fraction_identifier" + if normalized == "comment[technical replicate]": + return "technical_replicate" + if normalized == "comment[instrument]": + return "instrument_override" + return None + + +def _selected_file_lookup(selected_files: list[str]) -> dict[str, str]: + lookup = {} + for file_path in selected_files: + file_text = str(file_path) + lookup[file_text] = file_text + lookup[Path(file_text).name] = file_text + return lookup + + +def read_sdrf_user_metadata( + metadata_path: str | Path, + selected_files: list[str], +) -> tuple[list[dict], list[str]]: + """Read user-supplied SDRF fields for CLI runs.""" + selected_lookup = _selected_file_lookup(selected_files) + metadata_path = Path(metadata_path) + with metadata_path.open("r", newline="", encoding="utf-8-sig") as metadata_file: + reader = csv.DictReader(metadata_file, delimiter="\t") + if not reader.fieldnames: + raise ValueError(f"{metadata_path} does not contain a TSV header.") + + extra_columns = [] + internal_fields = {} + for header in reader.fieldnames: + cleaned_header = _text(header) + field = _user_field_for_header(cleaned_header) + if field is not None: + internal_fields[cleaned_header] = field + elif cleaned_header in AUTOMATIC_SDRF_HEADERS: + continue + else: + canonical = _normalized_extra_columns([cleaned_header]) + extra_columns.append(canonical[0] if canonical else cleaned_header) + + rows = [] + for source_row in reader: + row = {} + for header, value in source_row.items(): + cleaned_header = _text(header) + field = internal_fields.get(cleaned_header) + cleaned_value = _text(value) + if field == "file": + row[field] = selected_lookup.get(cleaned_value, cleaned_value) + elif field: + row[field] = cleaned_value + elif cleaned_header in AUTOMATIC_SDRF_HEADERS: + continue + else: + canonical = _normalized_extra_columns([cleaned_header]) + row[canonical[0] if canonical else cleaned_header] = cleaned_value + rows.append(row) + + return rows, _normalized_extra_columns(extra_columns) + + +def write_sdrf_cli_template( + output_path: str | Path, + selected_files: list[str], +) -> Path: + """Write a fillable TSV containing the SDRF fields CLI users must provide.""" + headers = [ + "RAW file", + *[label for _, label in USER_REQUIRED_FIELDS], + "comment[instrument]", + ] + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", newline="", encoding="utf-8") as output_file: + writer = csv.writer(output_file, delimiter="\t", lineterminator="\n") + writer.writerow(headers) + for file_path in selected_files: + base = Path(file_path).stem + writer.writerow( + [ + str(file_path), + base, + base, + "", + "not available", + "1", + "", + "", + "", + "1", + "1", + "", + ] + ) + return output_path + + def validate_sdrf_metadata( rows: list[dict], selected_files: list[str], @@ -255,7 +461,9 @@ def enrich_sdrf_rows_for_file( row = dict(user_row) row["file"] = str(file_path) row["data_file"] = Path(file_path).name - row["instrument"] = _known_text(row.get("instrument_override")) or _known_text(instrument) + row["instrument"] = normalize_instrument( + _known_text(row.get("instrument_override")) or _known_text(instrument) + ) row["acquisition_date"] = normalize_acquisition_date(acquisition_date) row["acquisition_method"] = normalize_acquisition_method(row.get("acquisition_method", "")) row["cleavage_agent"] = normalize_cleavage_agent(row.get("cleavage_agent", "")) @@ -263,6 +471,30 @@ def enrich_sdrf_rows_for_file( return enriched +def draft_sdrf_row_for_file( + file_path: str, + instrument: str, + acquisition_date, +) -> dict: + base = Path(file_path).stem + return { + "file": str(file_path), + "source_name": base, + "assay_name": base, + "organism": "", + "organism_part": "", + "biological_replicate": "", + "acquisition_method": "", + "label": "", + "instrument": normalize_instrument(instrument), + "cleavage_agent": "", + "fraction_identifier": "", + "technical_replicate": "", + "data_file": Path(file_path).name, + "acquisition_date": normalize_acquisition_date(acquisition_date), + } + + def write_sdrf( output_path: str | Path, rows: list[dict], @@ -301,6 +533,7 @@ def write_sdrf( "comment[technical replicate]", "comment[data file]", "comment[acquisition date]", + "comment[sdrf annotation tool]", "comment[sdrf version]", "comment[sdrf template]", *other_headers, @@ -324,6 +557,7 @@ def write_sdrf( "comment[technical replicate]": row.get("technical_replicate"), "comment[data file]": row.get("data_file"), "comment[acquisition date]": row.get("acquisition_date") or "not available", + "comment[sdrf annotation tool]": SDRF_ANNOTATION_TOOL, "comment[sdrf version]": SDRF_VERSION, "comment[sdrf template]": SDRF_TEMPLATE, } diff --git a/tests/test_sdrf_export.py b/tests/test_sdrf_export.py index 987385f..98ce4e3 100644 --- a/tests/test_sdrf_export.py +++ b/tests/test_sdrf_export.py @@ -4,10 +4,14 @@ from sdrf_export import ( available_sdrf_columns, + draft_sdrf_row_for_file, enrich_sdrf_rows_for_file, normalize_acquisition_date, + normalize_instrument, + read_sdrf_user_metadata, validate_sdrf_metadata, write_sdrf, + write_sdrf_cli_template, ) @@ -90,7 +94,10 @@ def test_enriches_and_writes_sdrf(self): self.assertEqual(len(lines), 3) self.assertIn("factor value[disease]", lines[0]) self.assertIn("NT=Data-dependent acquisition;AC=PRIDE:0000627", lines[1]) + self.assertIn("NT=Orbitrap Fusion Lumos;AC=MS:1002732", lines[1]) self.assertIn("NT=Lys-C;AC=MS:1001309", lines[2]) + self.assertIn("comment[sdrf annotation tool]", lines[0]) + self.assertIn("MetaXtract", lines[1]) self.assertEqual(len(lines[0].split("\t")), len(lines[1].split("\t"))) def test_unknown_raw_instrument_requires_an_override(self): @@ -108,6 +115,35 @@ def test_normalizes_thermo_creation_date(self): "2024-09-20T11:36:09", ) + def test_normalizes_common_thermo_instruments_to_cv_terms(self): + self.assertEqual( + normalize_instrument("Q Exactive HF-X Orbitrap"), + "NT=Q Exactive HF-X;AC=MS:1002877", + ) + self.assertEqual( + normalize_instrument("Thermo Scientific Orbitrap Exploris 480 Mass Spectrometer"), + "NT=Orbitrap Exploris 480;AC=MS:1003028", + ) + self.assertEqual( + normalize_instrument("custom prototype"), + "custom prototype", + ) + + def test_builds_draft_sdrf_row_with_only_raw_derived_values(self): + row = draft_sdrf_row_for_file( + "/data/small.RAW", + "LTQ FT", + "07/20/2005 14:44:22", + ) + + self.assertEqual(row["source_name"], "small") + self.assertEqual(row["assay_name"], "small") + self.assertEqual(row["data_file"], "small.RAW") + self.assertEqual(row["instrument"], "NT=LTQ FT;AC=MS:1000448") + self.assertEqual(row["acquisition_date"], "2005-07-20T14:44:22") + self.assertEqual(row["organism"], "") + self.assertEqual(row["acquisition_method"], "") + def test_offers_full_known_column_catalog(self): columns = available_sdrf_columns() self.assertGreaterEqual(len(columns), 300) @@ -167,6 +203,61 @@ def test_rejects_empty_or_unknown_added_columns(self): self.assertTrue(any("cannot be empty" in error for error in errors)) self.assertTrue(any("not a known SDRF column" in error for error in errors)) + def test_writes_and_reads_cli_sdrf_metadata_template(self): + with tempfile.TemporaryDirectory() as temp_dir: + template_path = Path(temp_dir) / "sdrf_input.tsv" + write_sdrf_cli_template(template_path, self.files) + header = template_path.read_text(encoding="utf-8").splitlines()[0] + template_path.write_text( + "\n".join( + [ + header, + "\t".join( + [ + "control.raw", + "control", + "control", + "homo sapiens", + "not available", + "1", + "DDA", + "label free sample", + "Trypsin", + "1", + "1", + "", + ] + ), + "\t".join( + [ + "treated.raw", + "treated", + "treated", + "homo sapiens", + "not available", + "1", + "DIA", + "label free sample", + "Trypsin", + "1", + "1", + "", + ] + ), + ] + ) + + "\n", + encoding="utf-8", + ) + + rows, extra_columns = read_sdrf_user_metadata(template_path, self.files) + + self.assertEqual(extra_columns, []) + self.assertEqual(rows[0]["file"], self.files[0]) + self.assertEqual(rows[0]["source_name"], "control") + self.assertEqual(rows[1]["acquisition_method"], "DIA") + self.assertEqual(validate_sdrf_metadata(rows, self.files), []) + if __name__ == "__main__": unittest.main() From 05727f81ff891ffbf975a4b5126ed2d6a461a5c6 Mon Sep 17 00:00:00 2001 From: Ahmad lutfi Date: Fri, 28 Aug 2026 19:19:48 +0200 Subject: [PATCH 8/8] check_box --- README.md | 11 ++++++++- cli_parser.py | 29 ++++++++++++++++++++++-- gui.py | 23 ++++++++++++++++--- plotly_visualizer.py | 54 +++++++++++++++++++++++++++++++++++++++----- 4 files changed, 105 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 418f3e8..e433353 100644 --- a/README.md +++ b/README.md @@ -400,7 +400,16 @@ Columns intentionally marked as Thermo-specific because they are RAW trailer fie Check the [documentation](Doc/Doc.pdf) for more details. #### Visualisation -Interactive Plotly HTML reports, MS1 and MS2 trends, and cross-sample overlays and boxplots. In the GUI, enable **Multi-sample comparison** and select any 2 or more of the loaded samples. In YAML/CLI runs, list the samples under `multi_comparison.samples` using 1-based indices such as `[1, 2, 4]`. +Interactive Plotly HTML reports, MS1 and MS2 trends, and cross-sample overlays and boxplots. + +Visualisation output cases: + +- One RAW file with visualisation enabled: MetaXtract writes only the per-file reports, for example `_MS1.html` and/or `_MS2.html`. +- Two or more RAW files with visualisation enabled: MetaXtract writes the per-file reports and also writes combined comparison reports, `MS1_compare.html` and/or `MS2_compare.html`, using all processed files. +- GUI with **Multi-sample comparison** enabled: the user selects any 2 or more loaded files, and `MS1_compare.html` / `MS2_compare.html` are generated only for that selected subset. +- YAML/CLI with `multi_comparison.enabled: true`: list the files to compare under `multi_comparison.samples` using 1-based input positions such as `[1, 2, 4]`; the comparison reports are generated for that subset. + +If visualisation is disabled, no per-file or comparison HTML reports are written. #### Runtime and memory logging For every processed RAW file, both the GUI log and CLI output report memory at the start and a final summary containing runtime, ending memory, sampled peak memory, and memory change. Each run also writes `runtime_summary_YYYYMMDD_HHMMSS.tsv` in the root output directory. The TSV contains one row per processed RAW file with status, runtime in seconds, start/end/peak memory in GB, and memory change in GB. Memory is the resident set size (RSS) of the MetaXtract process, so it includes Python, native libraries, and Thermo/.NET allocations used while that file is processed. diff --git a/cli_parser.py b/cli_parser.py index 3642fa2..556a1b7 100644 --- a/cli_parser.py +++ b/cli_parser.py @@ -5,6 +5,7 @@ import sys import threading import yaml +import math from datetime import datetime import csv, json from pathlib import Path @@ -14,6 +15,7 @@ from plotly_visualizer import ( PlotlyMS1Visualizer, PlotlyMS2Visualizer, + to_float, write_comparison_html_multi, write_comparison_html_with_boxplots, ) @@ -159,6 +161,22 @@ def trailer_value(trailer_data, output_label: str): return None +def selected_ion_intensity_value(raw_parser, scan_number: int, trailer_data) -> tuple[object, str]: + value = trailer_value(trailer_data, "Selected Ion Intensity") + numeric_value = to_float(value) + if numeric_value is not None and math.isfinite(numeric_value): + return value, "trailer" + + try: + value = raw_parser.GetPrecursorIntensityFromScanNumber(scan_number) + except Exception: + value = None + numeric_value = to_float(value) + if numeric_value is not None and math.isfinite(numeric_value): + return value, "computed" + return "N/A", "missing" + + def format_scan_window_mz_range(raw_parser, scan_number: int): n = raw_parser.GetNumberOfMassRangesFromScanNumber(scan_number) or 0 ranges = [] @@ -284,7 +302,10 @@ def extract_scan_header_to_csv( trailer_data = raw_parser.GetTrailerExtraInformaionEdited(scan_number) or {} for option in selected_options: - value = trailer_value(trailer_data, option) + if option in ("Selected Ion Intensity", "Precursor Intensity"): + value, _source = selected_ion_intensity_value(raw_parser, scan_number, trailer_data) + else: + value = trailer_value(trailer_data, option) if value is None: value = option_functions.get(option, lambda sn: "N/A")(scan_number) row.append(value) @@ -292,12 +313,16 @@ def extract_scan_header_to_csv( csv_writer.writerow(row) if graphical_representation and plotly_vis is not None: + selected_ion_intensity, selected_ion_source = selected_ion_intensity_value( + raw_parser, scan_number, trailer_data + ) plotly_vis.ms2_scans.append(scan_number) plotly_vis.ms2_data["Scan Start Time (min)"].append(raw_parser.GetRetentionTimeFromScanNumber(scan_number)) plotly_vis.ms2_data["Elapsed Scan Time (sec)"].append(raw_parser.GetElaspedScanTimeFromScanNumber(scan_number)) plotly_vis.ms2_data["Total Ion Current"].append(raw_parser.GetTICForScanNumber(scan_number)) plotly_vis.ms2_data["Total Number of Peaks"].append(raw_parser.GetNumPeaksForScanNumber(scan_number)) - plotly_vis.ms2_data["Selected Ion Intensity"].append(raw_parser.GetPrecursorIntensityFromScanNumber(scan_number)) + plotly_vis.ms2_data["Selected Ion Intensity"].append(selected_ion_intensity) + plotly_vis.ms2_data["Selected Ion Intensity Source"].append(selected_ion_source) plotly_vis.ms2_data["Charge State"].append(raw_parser.GetMS2ChargeFromScanNumber(scan_number)) plotly_vis.ms2_data["Ion Injection Time (ms)"].append(raw_parser.GetIonInjectionTimeFromScanNumber(scan_number)) plotly_vis.ms2_data.setdefault("Base Peak Intensity", []).append(raw_parser.GetBasePeakForScanNumber(scan_number)[1]) diff --git a/gui.py b/gui.py index 9357b22..922c7de 100644 --- a/gui.py +++ b/gui.py @@ -2,6 +2,7 @@ import csv import json +import math import os import re import signal @@ -246,6 +247,19 @@ def trailer_value(trailer_data, output_label: str): return None +def selected_ion_intensity_value(raw_parser, scan_number: int, trailer_data) -> tuple[object, str]: + value = trailer_value(trailer_data, "Selected Ion Intensity") + numeric_value = to_float(value) + if numeric_value is not None and math.isfinite(numeric_value): + return value, "trailer" + + value = safe_call(lambda: raw_parser.GetPrecursorIntensityFromScanNumber(scan_number)) + numeric_value = to_float(value) + if numeric_value is not None and math.isfinite(numeric_value): + return value, "computed" + return "N/A", "missing" + + class LogWindow(QDialog): def __init__(self, parent=None): @@ -432,7 +446,7 @@ def opt_value(opt: str): return bp[1] if isinstance(bp, (list, tuple)) and len(bp) >= 2 and bp[1] not in (None, "") else "N/A" if opt in ("Selected Ion Intensity", "Precursor Intensity"): - v = safe_call(lambda: raw_parser.GetPrecursorIntensityFromScanNumber(scan_number)) + v, _source = selected_ion_intensity_value(raw_parser, scan_number, trailer_data) return v if v not in (None, "") else "N/A" if opt in ("Scan Window m/z Range", "Mass Ranges"): @@ -481,7 +495,8 @@ def opt_value(opt: str): tnp = to_int(safe_call(lambda: raw_parser.GetNumPeaksForScanNumber(scan_number))) cs = to_int(td_get(trailer_data, "Charge State")) iit = to_float(td_get(trailer_data, "Ion Injection Time (ms)")) - prec_i = to_float(safe_call(lambda: raw_parser.GetPrecursorIntensityFromScanNumber(scan_number))) + prec_i_raw, prec_i_source = selected_ion_intensity_value(raw_parser, scan_number, trailer_data) + prec_i = to_float(prec_i_raw) if rt is None: rt = to_float(safe_call(lambda: raw_parser.GetRetentionTimeFromScanNumber(scan_number))) @@ -492,7 +507,8 @@ def opt_value(opt: str): if tnp is None: tnp = to_int(safe_call(lambda: raw_parser.GetNumPeaksForScanNumber(scan_number))) if prec_i is None: - prec_i = to_float(safe_call(lambda: raw_parser.GetPrecursorIntensityFromScanNumber(scan_number))) + prec_i_raw, prec_i_source = selected_ion_intensity_value(raw_parser, scan_number, trailer_data) + prec_i = to_float(prec_i_raw) if cs is None: cs = to_int(safe_call(lambda: raw_parser.GetMS2ChargeFromScanNumber(scan_number))) if iit is None: @@ -507,6 +523,7 @@ def opt_value(opt: str): plotly_vis.ms2_data["Total Ion Current"].append(tic or 0.0) plotly_vis.ms2_data["Total Number of Peaks"].append(tnp or 0) plotly_vis.ms2_data["Selected Ion Intensity"].append(prec_i or 0.0) + plotly_vis.ms2_data["Selected Ion Intensity Source"].append(prec_i_source) plotly_vis.ms2_data["Charge State"].append(cs or 0) plotly_vis.ms2_data["Ion Injection Time (ms)"].append(iit or 0.0) diff --git a/plotly_visualizer.py b/plotly_visualizer.py index 25c2878..a0aac57 100644 --- a/plotly_visualizer.py +++ b/plotly_visualizer.py @@ -77,6 +77,44 @@ def _log10p1(vals): arr = np.where(arr < 0, 0, arr) return np.log10(arr + 1.0) + +def _selected_ion_intensity_note(sources) -> str: + sources = list(sources or []) + if not sources: + return "" + trailer_count = sources.count("trailer") + computed_count = sources.count("computed") + missing_count = sources.count("missing") + if computed_count == 0 and missing_count == 0: + return "" + parts = [ + f"{trailer_count} scans used RAW trailer metadata", + f"{computed_count} scans used computed precursor-intensity fallback", + ] + if missing_count: + parts.append(f"{missing_count} scans had no usable value") + return "Selected Ion Intensity source: " + "; ".join(parts) + "." + + +def _add_bottom_note(fig: go.Figure, note: str) -> go.Figure: + if not note: + return fig + fig.add_annotation( + text=note, + xref="paper", + yref="paper", + x=0, + y=-0.22, + showarrow=False, + align="left", + xanchor="left", + yanchor="top", + font=dict(size=12, color="#4b5563"), + ) + fig.update_layout(margin=dict(b=100)) + return fig + + def make_boxplot_figure(title: str, y_label: str, sample_to_values: dict, log10p1: bool = False) -> go.Figure: fig = go.Figure() for sample, values in (sample_to_values or {}).items(): @@ -293,6 +331,7 @@ def __init__(self, single_file_name: str, output_dir: str): "Total Ion Current": [], "Total Number of Peaks": [], "Selected Ion Intensity": [], + "Selected Ion Intensity Source": [], "Charge State": [], "Ion Injection Time (ms)": [], "Base Peak Intensity": [], @@ -308,6 +347,9 @@ def _figs(self) -> List[_Fig]: tic = _farr(self.ms2_data["Total Ion Current"]) tnp = _farr(self.ms2_data["Total Number of Peaks"]) prec = _farr(self.ms2_data["Selected Ion Intensity"]) + prec_note = _selected_ion_intensity_note( + self.ms2_data.get("Selected Ion Intensity Source", []) + ) cs = _iarr(self.ms2_data["Charge State"]) iit_ms = _farr(self.ms2_data["Ion Injection Time (ms)"]) est = _farr(self.ms2_data["Elapsed Scan Time (sec)"]) @@ -333,29 +375,29 @@ def _figs(self) -> List[_Fig]: if prec.size: figs.append(_Fig( "Selected Ion Intensity vs RT (min)", - go.Figure( + _add_bottom_note(go.Figure( data=[go.Scatter(x=rt_min, y=prec, mode="lines", name=self.single_file_name)], layout=go.Layout(title="MS2 Selected Ion Intensity vs RT (min)", xaxis_title="RT (min)", yaxis_title="Selected Ion Intensity"), - ) + ), prec_note) )) if iit_ms.size and prec.size: iit_s = iit_ms / 1000.0 figs.append(_Fig( "2D hist: log10(Selected Ion Intensity) vs IIT (s)", - go.Figure( + _add_bottom_note(go.Figure( data=[go.Histogram2d(x=iit_s, y=_safe_log10(prec), nbinsx=60, nbinsy=60)], layout=go.Layout(title="MS2 log10(Selected Ion Intensity) vs IIT (s)", xaxis_title="IIT (s)", yaxis_title="log10(Selected Ion Intensity)"), - ) + ), prec_note) )) if tnp.size and prec.size: figs.append(_Fig( "2D hist: log10(Selected Ion Intensity) vs Total Peaks", - go.Figure( + _add_bottom_note(go.Figure( data=[go.Histogram2d(x=tnp, y=_safe_log10(prec), nbinsx=60, nbinsy=60)], layout=go.Layout(title="MS2 log10(Selected Ion Intensity) vs Total Peaks", xaxis_title="Total Peaks", yaxis_title="log10(Selected Ion Intensity)"), - ) + ), prec_note) )) if cs.size: