Functional tests

.github/workflows/test.yml

@@ -13,9 +13,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  POETRY_VERSION: "1.8.3"
+  PREFERRED_PYTHON_VERSION: "3.12"
+
 jobs:
-  cpu-tests:
-    name: Test on ${{ matrix.os }} with Python ${{ matrix.python-version }}
+  cpu-unit-tests:
+    name: Unit tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -26,48 +30,82 @@ jobs:
       - uses: actions/checkout@v4
       - uses: snok/install-poetry@v1
         with:
-          version: 1.8.3
+          version: ${{ env.POETRY_VERSION }}
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: poetry
       - name: Run unit tests
-        run: make test
+        run: make unit-tests
 
-  gpu-tests:
-    name: GPU test on ${{ matrix.os }} with Python ${{ matrix.python-version }}
+  gpu-unit-tests:
+    name: GPU unit tests
     runs-on:
       - self-hosted
       - gpu
-    strategy:
-      fail-fast: false
     steps:
       - uses: actions/checkout@v4
       - uses: snok/install-poetry@v1
         with:
-          version: 1.8.3
+          version: ${{ env.POETRY_VERSION }}
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: ${{ env.PREFERRED_PYTHON_VERSION }}
           cache: poetry
       - name: Run unit tests
-        run: make gpu-test
+        run: make gpu-unit-tests
+
+  cpu-functional-test:
+    name: Functional tests
+    runs-on:
+      - self-hosted
+      - functional-tests
+    needs: gpu-unit-tests
+    steps:
+      - uses: actions/checkout@v4
+      - uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PREFERRED_PYTHON_VERSION }}
+          cache: poetry
+      - name: Run functional tests
+        run: make functional-tests
 
-  multi-gpu-tests:
-    name: Multi-GPUs test on ${{ matrix.os }} with Python ${{ matrix.python-version }}
+  gpu-functional-test:
+    name: GPU functional tests
+    runs-on:
+      - self-hosted
+      - functional-tests
+    needs: cpu-functional-test
+    steps:
+      - uses: actions/checkout@v4
+      - uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PREFERRED_PYTHON_VERSION }}
+          cache: poetry
+      - name: Run functional tests
+        run: make gpu-functional-tests
+
+  multi-gpu-unit-tests:
+    name: Multi-GPUs unit tests
     runs-on:
       - self-hosted
       - multi-gpu
-    strategy:
-      fail-fast: false
     steps:
       - uses: actions/checkout@v4
       - uses: snok/install-poetry@v1
         with:
-          version: 1.8.3
+          version: ${{ env.POETRY_VERSION }}
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: ${{ env.PREFERRED_PYTHON_VERSION }}
           cache: poetry
+      - name: Cache issue (to solve)
+        run: rm -rf ~/.cache/pypoetry ~/.cache/pip
       - name: Run unit tests
-        run: make multi-gpu-test
+        run: make multi-gpu-unit-tests

Makefile

@@ -36,7 +36,7 @@ env: env.dev
 
 .PHONY: env.conda
 env.conda:
-	@$(CONDA) env create -p $(CONDA_ENV)
+	@$(CONDA) env create -p $(CONDA_ENV) -k
 
 .PHONY: env.dev
 env.dev:
@@ -83,17 +83,23 @@ install.dev: check.lock
 install.doc: check.lock
 	@$(POETRY) install --only docs
 
-## tests        : Run the unit tests
-.PHONY: test
-test: install
+## tests
+.PHONY: unit-tests
+unit-tests: install
 	@$(POETRY) run python -m pytest -v -m "not gpu and not multi_gpu" tests/unittests
 
-## gpu-tests    : Run only GPU unit tests
-.PHONY: gpu-test
-gpu-test: install
+.PHONY: gpu-unit-tests
+gpu-unit-tests: install
 	@$(POETRY) run python -m pytest -v -m "gpu" tests/unittests
 
-## multi-gpu-tests    : Run only Multi-GPUs unit tests
-.PHONY: multi-gpu-test
-multi-gpu-test: install
-	@$(POETRY) run python -m pytest -v -m "multi_gpu" tests/unittests
+.PHONY: multi-gpu-unit-tests
+multi-gpu-unit-tests: install
+	@$(POETRY) run python -m pytest -v -m "multi_gpu" tests/unittests
+
+.PHONY: functional-tests
+functional-tests: install
+	@$(POETRY) run python -m pytest -v -m "not gpu and not multi_gpu" --ref /localdrive10TB/users/ci-clinicadl/clinicadl_data_ci/data_ci tests/functional
+
+.PHONY: gpu-functional-tests
+gpu-functional-tests: install
+	@$(POETRY) run python -m pytest -v -m "gpu" --ref /localdrive10TB/users/ci-clinicadl/clinicadl_data_ci/data_ci tests/functional

clinicadl/callbacks/implemented/checks.py

@@ -25,6 +25,7 @@
     from clinicadl.data.datasets import Dataset
     from clinicadl.data.datasets.base import BaseDataset
     from clinicadl.io import Maps
+    from clinicadl.io.maps.utils import ModelDir
     from clinicadl.losses.types import LossType
     from clinicadl.models import Model
     from clinicadl.split import Split
@@ -73,14 +74,12 @@ def on_validate_start(self, **kwargs) -> None:
         self._check_batch.on_validate_start(**kwargs)
 
     def on_test_start(self, **kwargs) -> None:
-        self._check_inputs.on_test_start(**kwargs)
         self._check_dataframes.on_test_start(**kwargs)
         self._check_data_leakage.on_test_start(**kwargs)
         self._check_data_consistency.on_test_start(**kwargs)
         self._check_batch.on_test_start(**kwargs)
 
     def on_predict_start(self, **kwargs) -> None:
-        self._check_inputs.on_predict_start(**kwargs)
         self._check_dataframes.on_predict_start(**kwargs)
         self._check_data_consistency.on_predict_start(**kwargs)
         self._check_batch.on_predict_start(**kwargs)
@@ -95,63 +94,12 @@ def on_train_start(self, *, split: Split, **kwargs) -> None:
         """
         Checks the split index and that dataloaders have been instantiated in the splits.
         """
-        if split.train_loader is None:
-            raise RuntimeError(
-                "The split has no training dataloder defined. Please run 'build_train_loader'"
-            )
-        if split.val_loader is None:
-            raise RuntimeError(
-                "The split has no validation dataloder defined. Please run 'build_val_loader'"
-            )
+        split.train_loader
+        split.val_loader
 
     def on_resume(self, *, split: Split, **kwargs) -> None:
         self.on_train_start(split=split)
 
-    def on_test_start(
-        self,
-        *,
-        maps: Maps,
-        group_name: str,
-        model_checkpoint: str,
-        **kwargs,
-    ) -> None:
-        """
-        Checks if the checkpoint has already been tested on this group.
-        """
-        self._check_checkpoint(model_checkpoint, group_name, maps=maps, test=True)
-
-    def on_predict_start(
-        self,
-        *,
-        maps: Maps,
-        group_name: str,
-        model_checkpoint: str,
-        **kwargs,
-    ) -> None:
-        """
-        Checks if the group has already been predicted with this checkpoint.
-        """
-        self._check_checkpoint(model_checkpoint, group_name, maps=maps, test=False)
-
-    @staticmethod
-    def _check_checkpoint(
-        model_checkpoint: str, group_name: str, maps: Maps, test: bool
-    ):
-        split_idx, chkpt = maps.training.read_checkpoint_name(model_checkpoint)
-        if test:
-            dir_ = maps.test
-        else:
-            dir_ = maps.prediction
-
-        if (
-            split_idx in dir_.groups[group_name].results.splits_list
-            and chkpt in dir_.groups[group_name].results.splits[split_idx].models_list
-        ):
-            raise FileExistsError(
-                f"There are already some results for checkpoint '{chkpt}' in {dir_.groups[group_name].results.splits[split_idx].models[chkpt].path}. "
-                f"If you want to continue, please first delete the folder."
-            )
-
 
 class _CheckLosses:
     """
@@ -321,6 +269,9 @@ def on_train_start(self, *, maps: Maps, split: Split, **kwargs) -> None:
         Raises warnings if these objects differ or cannot be compared.
         """
         for split_idx in maps.training.splits_list:
+            if split_idx == split.index:
+                continue
+
             self._compare(
                 split.train_dataset,
                 maps.training.data.train.splits[split_idx].dataset_json,
@@ -440,23 +391,24 @@ def on_test_start(
         - error if the (participant, session) couples in the dataset are not the same as the group couples;
         - warning if the dataset doesn't match with the original test dataset (or cannot be compared).
         """
-        self._compare_participants_sessions(
-            dataloader.dataset,
-            maps.test.groups[group_name].data_tsv,
-            maps=maps,
-            stage="test",
-            group=group_name,
-        )
-
-        self._compare(
-            dataloader.dataset,
-            maps.test.groups[group_name].dataset_json,
-            getter=get_dataset_from_json_safely,
-            comparator=_compare_datasets,
-            new_group=group_name,
-            old_group=group_name,
-            stage="test",
-        )
+        if maps.test.groups[group_name].data_tsv.exists():
+            self._compare_participants_sessions(
+                dataloader.dataset,
+                maps.test.groups[group_name].data_tsv,
+                maps=maps,
+                stage="test",
+                group=group_name,
+            )
+        if maps.test.groups[group_name].dataset_json.exists():
+            self._compare(
+                dataloader.dataset,
+                maps.test.groups[group_name].dataset_json,
+                getter=get_dataset_from_json_safely,
+                comparator=_compare_datasets,
+                new_group=group_name,
+                old_group=group_name,
+                stage="test",
+            )
 
     def on_predict_start(
         self,
@@ -471,23 +423,24 @@ def on_predict_start(
         - error if the (participant, session) couples in the dataset are not the same as the group couples;
         - warning if the dataset doesn't match with the original prediction dataset (or cannot be compared).
         """
-        self._compare_participants_sessions(
-            dataloader.dataset,
-            maps.prediction.groups[group_name].data_tsv,
-            maps=maps,
-            stage="prediction",
-            group=group_name,
-        )
-
-        self._compare(
-            dataloader.dataset,
-            maps.prediction.groups[group_name].dataset_json,
-            getter=get_dataset_from_json_safely,
-            comparator=_compare_datasets,
-            new_group=group_name,
-            old_group=group_name,
-            stage="prediction",
-        )
+        if maps.prediction.groups[group_name].data_tsv.exists():
+            self._compare_participants_sessions(
+                dataloader.dataset,
+                maps.prediction.groups[group_name].data_tsv,
+                maps=maps,
+                stage="prediction",
+                group=group_name,
+            )
+        if maps.prediction.groups[group_name].dataset_json.exists():
+            self._compare(
+                dataloader.dataset,
+                maps.prediction.groups[group_name].dataset_json,
+                getter=get_dataset_from_json_safely,
+                comparator=_compare_datasets,
+                new_group=group_name,
+                old_group=group_name,
+                stage="prediction",
+            )
 
     @staticmethod
     def _compare_participants_sessions(