Skip to content

Commit c823287

Browse files
authored
Merge pull request #188 from Roestlab/osw_parquet_chrom_reader
OpenSWATH XIC parquet reader
2 parents a0e48a9 + fc30b54 commit c823287

24 files changed

Lines changed: 6459 additions & 22 deletions

docs/API.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ Classes
9090

9191
ResultsLoader
9292
MzMLDataLoader
93+
OpenSwathXICParquetLoader
9394
SqMassLoader
9495
XICParquetDataLoader
9596
SpectralLibraryLoader
@@ -125,10 +126,12 @@ Classes
125126
SqMassDataAccess
126127
MzMLDataAccess
127128
OSWDataAccess
129+
OpenSwathXICParquetAccess
128130
ResultsTSVDataAccess
129131
TransitionPQPDataAccess
130132
TransitionTSVDataAccess
131133
XICParquetDataAccess
134+
132135

133136

134137
Abstract Classes
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
"""
2+
massdash/loaders/OpenSwathXICParquetLoader
3+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4+
"""
5+
6+
7+
from typing import List, Dict, Union
8+
from os.path import basename
9+
from pandas.core.api import DataFrame as DataFrame
10+
import pandas as pd
11+
12+
# Loaders
13+
from .GenericChromatogramLoader import GenericChromatogramLoader
14+
from .ResultsLoader import ResultsLoader
15+
from .access import OpenSwathXICParquetAccess
16+
# Structs
17+
from ..structs import TransitionGroup, TransitionGroupCollection
18+
# Utils
19+
from ..util import LOGGER
20+
21+
class OpenSwathXICParquetLoader(GenericChromatogramLoader):
22+
23+
'''
24+
Class for loading Chromatograms and peak features from XIC (PyProphet) parquet files and a results files.
25+
Inherits from GenericChromatogramLoader
26+
'''
27+
28+
def __init__(self, **kwargs):
29+
super().__init__(**kwargs)
30+
self.dataAccess = [OpenSwathXICParquetAccess(f) for f in self.dataFiles]
31+
32+
@ResultsLoader.cache_results
33+
def loadTransitionGroupsDf(self, pep_id: str, charge: int) -> pd.DataFrame:
34+
columns=['run_name', 'rt', 'intensity', 'annotation']
35+
out = {}
36+
for t in self.dataAccess:
37+
38+
df = t.getChromatogramDfFromSequenceAndCharge(pep_id, charge)
39+
# only add if there is data
40+
if not df.empty:
41+
out[t.runName] = df
42+
else:
43+
LOGGER.warning(f"Warning: no data found for peptide in transition file {t.filename}")
44+
45+
if out == {}:
46+
return pd.DataFrame(columns=columns)
47+
else:
48+
return pd.concat(out).reset_index().drop('level_1', axis=1).rename(columns=dict(level_0='run'))
49+
50+
@ResultsLoader.cache_results
51+
def loadTransitionGroups(self, pep_id: str, charge: int, runNames: Union[None, str, List[str]] =None) -> Dict[str, TransitionGroupCollection]:
52+
'''
53+
Loads the transition group for a given peptide ID and charge across all files
54+
Args:
55+
pep_id (str): Peptide ID
56+
charge (int): Charge
57+
runNames (None | str | List[str]): Name of the run to extract the transition group from. If None, all runs are extracted. If str, only the specified run is extracted. If List[str], only the specified runs are extracted.
58+
Returns:
59+
Dict[str, TransitionGroup]: Dictionary of TransitionGroups, with keys as sqMass filenames
60+
'''
61+
62+
out = TransitionGroupCollection()
63+
64+
def _assembleTransitionGroup(t):
65+
chroms = t.getChromatogramsFromSequenceAndCharge(pep_id, charge)
66+
precursorChroms = [i for i in chroms if 'precursor' in i.label.lower()]
67+
transitionChroms = [i for i in chroms if 'precursor' not in i.label.lower()]
68+
if len(precursorChroms) == 0 and len(transitionChroms) == 0: # do not create a transition group if there are no chromatograms
69+
return None
70+
else:
71+
return TransitionGroup(precursorChroms, transitionChroms, pep_id, charge)
72+
73+
if runNames is None:
74+
for t in self.dataAccess:
75+
out[t.runName] = _assembleTransitionGroup(t)
76+
elif isinstance(runNames, str):
77+
t = self.dataAccess[self.runNames.index(runNames)]
78+
out[runNames] = _assembleTransitionGroup(t)
79+
elif isinstance(runNames, list):
80+
out = TransitionGroupCollection()
81+
for r in runNames:
82+
for t in self.dataAccess:
83+
if t.runName == r:
84+
out[t.runName] = _assembleTransitionGroup(t)
85+
else:
86+
raise ValueError("runName must be none, a string or list of strings")
87+
88+
# if there are no chromatograms, return none
89+
if all([i is None for i in out.values()]):
90+
LOGGER.warning(f"No chromatograms found for peptide {pep_id} with charge {charge} in any of the runs")
91+
return None
92+
else:
93+
return out

massdash/loaders/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from .GenericChromatogramLoader import GenericChromatogramLoader
99
from .GenericSpectrumLoader import GenericSpectrumLoader
1010
from .MzMLDataLoader import MzMLDataLoader
11+
from .OpenSwathXICParquetLoader import OpenSwathXICParquetLoader
1112
from .ResultsLoader import ResultsLoader
1213
from .SpectralLibraryLoader import SpectralLibraryLoader
1314
from .SqMassLoader import SqMassLoader
@@ -17,6 +18,7 @@
1718
"GenericChromatogramLoader",
1819
"GenericSpectrumLoader",
1920
"MzMLDataLoader",
21+
"OpenSwathXICParquetLoader",
2022
"ResultsLoader",
2123
"SpectralLibraryLoader",
2224
"SqMassLoader",
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
"""
2+
massdash/loaders/access/OpenSwathXICParquetAccess.py
3+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4+
"""
5+
6+
#!/usr/bin/python
7+
# -*- coding: utf-8 -*-
8+
from typing import List
9+
from collections import OrderedDict
10+
import pyopenms as po
11+
import sqlite3
12+
import pandas as pd
13+
from pathlib import Path
14+
import pyarrow.dataset as ds
15+
16+
# Structs
17+
from ...structs.Chromatogram import Chromatogram
18+
# Utils
19+
from ...util import decodeCompressedArray
20+
21+
class OpenSwathXICParquetAccess:
22+
23+
def __init__(self, filename):
24+
self.filename = filename
25+
self.runName = str(Path(filename).stem)
26+
self.parquet = ds.dataset(filename)
27+
28+
def getChromatogramsFromSequenceAndCharge(self, sequence: str, charge: int):
29+
"""
30+
Get chromatograms for a given peptide sequence and charge
31+
"""
32+
df = self.parquet.scanner(
33+
columns=['RT_DATA', 'INTENSITY_DATA', 'RT_COMPRESSION', 'INTENSITY_COMPRESSION', 'TRANSITION_ORDINAL', 'TRANSITION_TYPE', 'PRODUCT_CHARGE', 'NATIVE_ID'],
34+
filter=(
35+
(ds.field("MODIFIED_SEQUENCE") == sequence) &
36+
(ds.field("PRECURSOR_CHARGE") == charge)
37+
)
38+
).to_table().to_pandas()
39+
40+
# Create an ANNOTATION column, is the annotation for transitions and the native ID for precursors
41+
mask = ~df['PRODUCT_CHARGE'].isnull()
42+
df.loc[mask, 'ANNOTATION'] = (df.loc[mask, 'TRANSITION_TYPE'] +
43+
df.loc[mask, 'TRANSITION_ORDINAL'].astype(int).astype(str) +
44+
'^' +
45+
df.loc[mask, 'PRODUCT_CHARGE'].astype(int).astype(str))
46+
df.loc[~mask, 'ANNOTATION'] = df.loc[~mask, 'NATIVE_ID']
47+
48+
chroms = []
49+
for _, row in df.iterrows():
50+
rt_data = decodeCompressedArray(row['RT_DATA'], row['RT_COMPRESSION'])
51+
intensity_data = decodeCompressedArray(row['INTENSITY_DATA'], row['INTENSITY_COMPRESSION'])
52+
chroms.append(Chromatogram(rt_data, intensity_data, row['ANNOTATION']))
53+
54+
return chroms
55+
56+
def getChromatogramDfFromSequenceAndCharge(self, sequence: str, charge: int) -> pd.DataFrame:
57+
'''
58+
Get chromatogram data as a dataframe
59+
'''
60+
chroms = self.getChromatogramsFromSequenceAndCharge(sequence, charge)
61+
chroms_df = []
62+
for c in chroms:
63+
chroms_df.append(c.toPandasDf())
64+
65+
if len(chroms_df) == 0:
66+
return pd.DataFrame(columns=['rt', 'intensity', 'annotation'])
67+
else:
68+
return pd.concat(chroms_df)
69+
70+
def __str__(self):
71+
return f"<OpenSwathXICParquetAccess(filename={self.filename})>"
72+
73+
def __repr__(self):
74+
return f"OpenSwathXICParquetAccess(filename={self.filename})"

massdash/loaders/access/SqMassDataAccess.py

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
# Structs
5454
from ...structs.Chromatogram import Chromatogram
5555
# Utils
56-
from ...util import check_sqlite_column_in_table, check_sqlite_table
56+
from ...util import decodeCompressedArray
5757

5858
class SqMassDataAccess:
5959

@@ -187,27 +187,7 @@ def _returnDataForChromatogram(self, data):
187187

188188
for chr_id, compr, data_type, d in data:
189189
result = []
190-
191-
if compr == 1:
192-
tmp = zlib.decompress(d)
193-
result = struct.unpack("<%sd" % (len(tmp) // 8), tmp)
194-
195-
if compr == 5:
196-
# tmp = [ord(q) for q in zlib.decompress(d)]
197-
tmp = bytearray( zlib.decompress(d) )
198-
if len(tmp) > 0:
199-
numpress_config.setCompression('linear')
200-
po.MSNumpressCoder().decodeNP(base64.b64encode(tmp), result, False, numpress_config)
201-
else:
202-
result = [0]
203-
if compr == 6:
204-
# tmp = [ord(q) for q in zlib.decompress(d)]
205-
tmp = bytearray( zlib.decompress(d) )
206-
if len(tmp) > 0:
207-
numpress_config.setCompression('slof')
208-
po.MSNumpressCoder().decodeNP(base64.b64encode(tmp), result, False, numpress_config)
209-
else:
210-
result = [0]
190+
result = decodeCompressedArray(d, compr)
211191

212192
if len(result) == 0:
213193
result = [ 0 ]

massdash/loaders/access/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from .GenericResultsAccess import GenericResultsAccess
99
from .MzMLDataAccess import MzMLDataAccess
1010
from .OSWDataAccess import OSWDataAccess
11+
from .OpenSwathXICParquetAccess import OpenSwathXICParquetAccess
1112
from .ResultsTSVDataAccess import ResultsTSVDataAccess
1213
from .SqMassDataAccess import SqMassDataAccess
1314
from .TransitionPQPDataAccess import TransitionPQPDataAccess
@@ -17,6 +18,7 @@
1718
__all__ = [ "GenericResultsAccess",
1819
"MzMLDataAccess",
1920
"OSWDataAccess",
21+
"OpenSwathXICParquetAccess",
2022
"ResultsTSVDataAccess",
2123
"SqMassDataAccess",
2224
"TransitionPQPDataAccess",

massdash/util.py

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,3 +579,50 @@ def in_notebook() -> bool:
579579
except AttributeError:
580580
return False
581581
return True
582+
583+
584+
def decodeCompressedArray(data, compr):
585+
'''
586+
Decodes a compressed array using the specified compression method.
587+
588+
compression methods:
589+
- 0 = no compression
590+
- 1 = zlib
591+
- 5 = numpress linear + zlib
592+
- 6 = numpress slof + zlib
593+
594+
Docstring for decodeArray
595+
596+
:param data: Description
597+
:param compr: Description
598+
'''
599+
600+
import pyopenms as po
601+
import zlib
602+
import struct
603+
import base64
604+
numpress_config = po.NumpressConfig()
605+
result = []
606+
if compr == 0:
607+
return data
608+
if compr == 1:
609+
tmp = zlib.decompress(data)
610+
return struct.unpack("<%sd" % (len(tmp) // 8), tmp)
611+
elif compr == 5:
612+
tmp = bytearray(zlib.decompress(data))
613+
if len(tmp) > 0:
614+
numpress_config.setCompression('linear')
615+
po.MSNumpressCoder().decodeNP(base64.b64encode(tmp), result, False, numpress_config)
616+
return result
617+
else:
618+
return [0]
619+
elif compr == 6:
620+
tmp = bytearray( zlib.decompress(data) )
621+
if len(tmp) > 0:
622+
numpress_config.setCompression('slof')
623+
po.MSNumpressCoder().decodeNP(base64.b64encode(tmp), result, False, numpress_config)
624+
return result
625+
else:
626+
return [0]
627+
else:
628+
raise Exception(f"Compression type {compr} not supported")

0 commit comments

Comments
 (0)