removed global ids (both in usage and storage)

Author: MichelDierckx

TreeMS2/groups/group.py

@@ -15,9 +15,6 @@ def __init__(self, group_name: str):
         self.failed_parsed = 0
         self.failed_processed = 0
 
-        self.begin = 0
-        self.end = 0
-
     def set_id(self, file_id: int):
         self._id = file_id
 
@@ -42,18 +39,6 @@ def add(self, peak_file: PeakFile) -> PeakFile:
     def get_peak_file(self, peak_file_id):
         return self._peak_files[peak_file_id]
 
-    def update(self, begin_id):
-        self.begin = begin_id
-        self.end = begin_id + self.total_spectra - 1
-        cur_id = self.begin
-        for peak_file in self._peak_files:
-            cur_id = peak_file.update(cur_id)
-        return self.end + 1
-
-    def get_global_id(self, peak_file_id: int, spectrum_id: int) -> int:
-        global_id = self._peak_files[peak_file_id].get_global_id(spectrum_id)
-        return global_id
-
     def total_valid_spectra(self) -> int:
         return self.total_spectra - self.failed_parsed - self.failed_processed
 
@@ -64,8 +49,6 @@ def to_dict(self) -> Dict[str, Any]:
             "total_spectra": self.total_spectra,
             "failed_parsed": self.failed_parsed,
             "failed_processed": self.failed_processed,
-            "begin": self.begin,
-            "end": self.end,
             "files": [file.to_dict() for file in self._peak_files],
         }
 
@@ -76,8 +59,6 @@ def from_dict(cls, data: Dict[str, Any]) -> "Group":
         group.total_spectra = data["total_spectra"]
         group.failed_parsed = data["failed_parsed"]
         group.failed_processed = data["failed_processed"]
-        group.begin = data["begin"]
-        group.end = data["end"]
         for file in data["files"]:
             _, file_extension = os.path.splitext(file["filename"])
             match file_extension:
@@ -86,7 +67,3 @@ def from_dict(cls, data: Dict[str, Any]) -> "Group":
                 case _:
                     continue
         return group
-
-    def __repr__(self) -> str:
-        files_repr = "\n\t".join([repr(file) for file in self._peak_files])
-        return f"{self.__class__.__name__}(id={self._id}, [{self.begin}, {self.end}]):\n\t{files_repr}"

TreeMS2/groups/groups.py

@@ -21,9 +21,6 @@ def __init__(self):
         self.failed_parsed = 0
         self.failed_processed = 0
 
-        self.begin = 0
-        self.end = 0
-
     def add(self, group: Group) -> Group:
         for g in self._groups:
             if g.get_group_name() == group.get_group_name():
@@ -50,22 +47,6 @@ def get_group(self, group_id: int) -> Group:
     def get_group_ids(self) -> List[int]:
         return [group.get_id() for group in self._groups]
 
-    def update(self):
-        self.end = self.total_spectra - 1
-        cur_id = self.begin
-        for group in self._groups:
-            cur_id = group.update(begin_id=cur_id)
-
-    def get_global_id(self, group_id: int, file_id, spectrum_id) -> int:
-        global_id = self._groups[group_id].get_global_id(file_id, spectrum_id)
-        return global_id
-
-    def get_group_id_from_global_id(self, global_id: int) -> int:
-        for group in self._groups:
-            if group.begin <= global_id <= group.end:
-                return group.get_id()
-        raise ValueError(f"Global id '{global_id}' does not belong to any group.")
-
     def total_valid_spectra(self) -> int:
         return self.total_spectra - self.failed_parsed - self.failed_processed
 
@@ -145,8 +126,6 @@ def to_dict(self) -> Dict[str, Any]:
             "total_spectra": self.total_spectra,
             "failed_parsed": self.failed_parsed,
             "failed_processed": self.failed_processed,
-            "begin": self.begin,
-            "end": self.end,
             "groups": [group.to_dict() for group in self._groups],
         }
 
@@ -167,8 +146,7 @@ def from_dict(cls, data: Dict[str, Any]) -> Optional["Groups"]:
             groups.total_spectra = data["total_spectra"]
             groups.failed_parsed = data["failed_parsed"]
             groups.failed_processed = data["failed_processed"]
-            groups.begin = data["begin"]
-            groups.end = data["end"]
+
             groups._groups = [Group.from_dict(group) for group in data["groups"]]
             return groups
         except (KeyError, TypeError, AttributeError):
@@ -184,7 +162,3 @@ def load(cls, path: str) -> Optional["Groups"]:
             return cls.from_dict(data)
         except (json.JSONDecodeError, OSError, PermissionError):
             return None  # Return None if the file is unreadable or corrupted
-
-    def __repr__(self) -> str:
-        groups_repr = "\n\t".join([repr(group) for group in self._groups])
-        return f"{self.__class__.__name__}({self.get_size()} groups, {self.get_nr_files()} files, [{self.begin}, {self.end}]):\n\t{groups_repr}"

TreeMS2/groups/peak_file/peak_file.py

@@ -15,8 +15,6 @@ def __init__(self, file_path: str):
         self.failed_parsed = 0
         self.failed_processed = 0
 
-        self.begin = 0
-        self.end = 0
         self.filtered: List[int] = []
 
     @abstractmethod
@@ -39,15 +37,6 @@ def set_group_id(self, group_id: int):
     def get_group_id(self):
         return self._group_id
 
-    def update(self, begin_id: int):
-        self.begin = begin_id
-        self.end = begin_id + self.total_spectra - 1
-        self.filtered = [x + self.begin for x in self.filtered]
-        return self.end + 1
-
-    def get_global_id(self, spectrum_id: int) -> int:
-        return self.begin + spectrum_id
-
     def total_valid_spectra(self) -> int:
         return self.total_spectra - self.failed_parsed - self.failed_processed
 
@@ -58,8 +47,6 @@ def to_dict(self) -> Dict[str, Any]:
             "total_spectra": self.total_spectra,
             "failed_parsed": self.failed_parsed,
             "failed_processed": self.failed_processed,
-            "begin": self.begin,
-            "end": self.end,
             "filtered": self.filtered,
         }
 
@@ -70,10 +57,5 @@ def from_dict(cls, data: Dict[str, Any]) -> "PeakFile":
         peak_file.total_spectra = data["total_spectra"]
         peak_file.failed_parsed = data["failed_parsed"]
         peak_file.failed_processed = data["failed_processed"]
-        peak_file.begin = data["begin"]
-        peak_file.end = data["end"]
         peak_file.filtered = data["filtered"]
         return peak_file
-
-    def __repr__(self) -> str:
-        return f"{self.__class__.__name__}(id={self._id}, filepath={self.file_path}, [{self.begin}, {self.end}])"

TreeMS2/similarity_matrix/filters/mask_filter.py

@@ -21,18 +21,3 @@ def apply(self, similarity_matrix: SimilarityMatrix):
     @abstractmethod
     def construct_mask(self, similarity_matrix: SimilarityMatrix) -> SpectraMatrix:
         pass
-
-    @abstractmethod
-    def write_filter_statistics(self, target_dir: str, total_spectra: int):
-        pass
-
-    @abstractmethod
-    def save_mask(self, target_dir: str):
-        pass
-
-    @abstractmethod
-    def save_mask_global(self, target_dir: str, total_spectra: int):
-        pass
-
-    def __repr__(self) -> str:
-        return f"{self.__class__.__name__}()"

TreeMS2/similarity_matrix/filters/precursor_mz_filter.py

@@ -1,10 +1,6 @@
-import os
-
 import numpy as np
-import pandas as pd
 from scipy.sparse import csr_matrix
 
-from TreeMS2.groups.groups import Groups
 from TreeMS2.logger_config import get_logger
 from TreeMS2.similarity_matrix.filters.mask_filter import MaskFilter
 from TreeMS2.similarity_matrix.similarity_matrix import SimilarityMatrix
@@ -15,11 +11,10 @@
 
 
 class PrecursorMzFilter(MaskFilter):
-    def __init__(self, groups: Groups, vector_store: VectorStore,
+    def __init__(self, vector_store: VectorStore,
                  precursor_mz_window: float):
         self.precursor_mz_window = precursor_mz_window
         self.vector_store = vector_store
-        self.groups = groups
         super().__init__(None)
 
     def construct_mask(self, similarity_matrix: SimilarityMatrix) -> SpectraMatrix:
@@ -42,67 +37,3 @@ def construct_mask(self, similarity_matrix: SimilarityMatrix) -> SpectraMatrix:
                        dtype=np.bool_)
         mask = SpectraMatrix(m)
         return mask
-
-    def write_filter_statistics(self, target_dir: str, total_spectra):
-        if self.mask is None:
-            raise ValueError("No mask has been constructed")
-
-        rows, cols = self.mask.matrix.nonzero()
-
-        row_ids = self.vector_store.get_data(rows, ["global_id"])["global_id"].to_numpy(dtype=np.int32)
-        col_ids = self.vector_store.get_data(cols, ["global_id"])["global_id"].to_numpy(dtype=np.int32)
-        m = csr_matrix((self.mask.matrix.data, (row_ids, col_ids)), shape=(total_spectra, total_spectra),
-                       dtype=np.bool_)
-
-        nr_groups = self.groups.get_size()
-        s = np.zeros((nr_groups, nr_groups), dtype=np.uint64)
-        # loop over groups representing the rows in s
-        for row_group in self.groups.get_groups():
-            row_group_id = row_group.get_id()
-            # retrieve id of first and last spectrum in group
-            row_begin = row_group.begin
-            row_end = row_group.end
-            # loop over groups representing the columns in s
-            for col_group in self.groups.get_groups():
-                col_group_id = col_group.get_id()
-                # retrieve id of first and last spectrum in group
-                col_begin = col_group.begin
-                col_end = col_group.end
-                # count the number of spectra that have been filtered between group A and group B
-                filtered = m[row_begin:row_end + 1, col_begin:col_end + 1].nnz
-                s[row_group_id, col_group_id] = filtered
-
-        # create a dataframe
-        group_names = [group.get_group_name() for group in self.groups.get_groups()]
-        df = pd.DataFrame(s, index=group_names, columns=group_names)
-
-        # create path
-        filters_dir = os.path.join(target_dir, "filters")
-        os.makedirs(filters_dir, exist_ok=True)
-        path = os.path.join(filters_dir, "precursor_mz.txt")
-
-        # write statistics to disk
-        with open(path, 'w') as f:
-            # Write a header explanation
-            f.write(
-                f"Number of spectra considered similar between each pair of groups with a precursor m/z difference larger than {self.precursor_mz_window}:\n\n")
-            # Write the matrix
-            f.write(df.to_string())
-        logger.info(
-            f"Overview of the number of similarities filtered due to precursor m/z difference written to '{path}'")
-        return s
-
-    def save_mask(self, target_dir: str):
-        # save mask
-        path = self.mask.write(os.path.join(target_dir, "precursor_mz"))
-        logger.info(f"Precursor mz mask has been written to '{path}'.")
-        return path
-
-    def save_mask_global(self, target_dir: str, total_spectra: int):
-        # save mask
-        path = self.mask.write_global(os.path.join(target_dir, "precursor_mz_global"), total_spectra, self.vector_store)
-        logger.info(f"Precursor mz mask has been written to '{path}'.")
-        return path
-
-    def __repr__(self) -> str:
-        return f"{self.__class__.__name__}(precursor_mz_window={self.precursor_mz_window:.3f})"

TreeMS2/similarity_matrix/pipeline.py

@@ -4,7 +4,6 @@
 from .filters.mask_filter import MaskFilter
 from .filters.precursor_mz_filter import PrecursorMzFilter
 from .similarity_matrix import SimilarityMatrix
-from ..groups.groups import Groups
 from ..vector_store.vector_store import VectorStore
 
 logger = get_logger(__name__)
@@ -35,11 +34,11 @@ def __repr__(self) -> str:
 class SimilarityMatrixPipelineFactory:
     @staticmethod
     def create_pipeline(
-            groups: Groups, vector_store: VectorStore,
+            vector_store: VectorStore,
             precursor_mz_window: Optional[float]) -> SimilarityMatrixPipeline:
         mask_filters = []
         if precursor_mz_window is not None:
             mask_filters.append(
-                PrecursorMzFilter(groups=groups, vector_store=vector_store, precursor_mz_window=precursor_mz_window))
+                PrecursorMzFilter(vector_store=vector_store, precursor_mz_window=precursor_mz_window))
         pipeline = SimilarityMatrixPipeline(mask_filters=mask_filters)
         return pipeline

TreeMS2/similarity_matrix/similarity_matrix.py

@@ -1,10 +1,9 @@
 import numpy as np
 import numpy.typing as npt
-from scipy.sparse import csr_matrix, load_npz
+from scipy.sparse import csr_matrix
 
 from TreeMS2.logger_config import get_logger
 from TreeMS2.similarity_matrix.spectra_matrix import SpectraMatrix
-from TreeMS2.vector_store.vector_store import VectorStore
 
 logger = get_logger(__name__)
 
@@ -17,25 +16,3 @@ def __init__(self, *args, similarity_threshold: float):
 
     def update(self, data: npt.NDArray[np.bool_], rows: npt.NDArray[np.int64], cols: npt.NDArray[np.int64]):
         self.matrix += csr_matrix((data, (rows, cols)), self.matrix.shape)
-
-    def write(self, path: str) -> str:
-        path = super().write(path)
-        return path
-
-    def write_global(self, path: str, total_spectra: int, vector_store: VectorStore):
-        path = super().write_global(path, total_spectra, vector_store)
-        return path
-
-    @classmethod
-    def load_with_threshold(cls, path: str, similarity_threshold: float) -> 'SimilarityMatrix':
-        # Load the matrix from file
-        matrix = load_npz(path)
-        # Create an instance of SimilarityMatrix with the loaded matrix and threshold
-        return cls(matrix, similarity_threshold=similarity_threshold)
-
-    def __sub__(self, other):
-        if isinstance(other, SpectraMatrix):
-            # Perform the subtraction and return a SimilarityMatrix
-            return SimilarityMatrix(self.matrix - other.matrix, similarity_threshold=self.similarity_threshold)
-        else:
-            raise ValueError("Subtraction is only supported between SimilarityMatrix and SpectraMatrix instances.")

TreeMS2/similarity_matrix/spectra_matrix.py

@@ -1,7 +1,5 @@
 import numpy as np
-from scipy.sparse import csr_matrix, save_npz, load_npz
-
-from TreeMS2.vector_store.vector_store import VectorStore
+from scipy.sparse import csr_matrix
 
 
 class SpectraMatrix:
@@ -23,29 +21,5 @@ def __init__(self, *args):
         else:
             raise ValueError("Invalid arguments for initializing SpectraMatrix.")
 
-    def nr_bytes(self):
-        return self.matrix.data.nbytes + self.matrix.indptr.nbytes + self.matrix.indices.nbytes
-
-    def write(self, path: str) -> str:
-        save_npz(path, self.matrix)
-        return path
-
-    def write_global(self, path: str, total_spectra: int, vector_store: VectorStore):
-        rows, cols = self.matrix.nonzero()
-        row_ids = vector_store.get_data(rows, ["global_id"])["global_id"].to_numpy(dtype=np.int32)
-        col_ids = vector_store.get_data(cols, ["global_id"])["global_id"].to_numpy(dtype=np.int32)
-        m = csr_matrix((self.matrix.data, (row_ids, col_ids)), shape=(total_spectra, total_spectra), dtype=np.bool_)
-        save_npz(path, m)
-        return path
-
     def subtract(self, spectra_matrix: 'SpectraMatrix'):
         self.matrix -= spectra_matrix.matrix
-
-    @classmethod
-    def load(cls, path: str) -> 'SpectraMatrix':
-        # Load the sparse matrix
-        matrix = load_npz(path)
-        # Create a new instance of SpectraMatrix
-        instance = cls(matrix)
-
-        return instance

TreeMS2/similarity_sets.py

@@ -4,7 +4,6 @@
 import numpy as np
 import numpy.typing as npt
 import pandas as pd
-from scipy.sparse import csr_matrix
 
 from TreeMS2.groups.groups import Groups
 from TreeMS2.logger_config import get_logger