QA eval pipeline for retrieval

nemo_retriever/examples/eval_sweep.yaml

@@ -0,0 +1,65 @@
+# QA Evaluation Sweep Configuration
+#
+# Usage (from repo root):
+#   export GEN_API_KEY="your-api-key"    # for generator-b (any OpenAI-compatible provider)
+#   export GEN_API_BASE="https://your-endpoint/v1"
+#   export NVIDIA_API_KEY="nvapi-..."    # for NIM models (judge, nemotron generators)
+#   retriever eval run --config nemo_retriever/examples/eval_sweep.yaml
+#
+# Secrets use ${VAR} expansion -- they are resolved from environment
+# variables at load time.  Never commit real keys in this file.
+#
+# All paths below are relative to the working directory (repo root).
+
+dataset:
+  source: "csv:data/bo767_annotations.csv"
+
+retrieval:
+  type: "file"
+  file_path: "data/eval/bo767_retrieval_fullpage.json"
+
+# Define models once, reference by name in evaluations.
+# Any model can serve as generator or judge.
+# api_base is any OpenAI-compatible endpoint (NIM, vLLM, Ollama, etc.).
+models:
+  nemotron-super-49b:
+    model: "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5"
+    api_key: "${NVIDIA_API_KEY}"
+
+  generator-b:
+    model: "openai/my-org/my-model"
+    api_base: "${GEN_API_BASE}"
+    api_key: "${GEN_API_KEY}"
+
+  nemotron-nano-30b:
+    model: "nvidia_nim/nvidia/nemotron-3-nano-30b-a3b"
+    api_key: "${NVIDIA_API_KEY}"
+
+  mixtral-judge:
+    model: "nvidia_nim/mistralai/mixtral-8x22b-instruct-v0.1"
+    api_key: "${NVIDIA_API_KEY}"
+
+# Each evaluation picks a generator + judge by name.
+# "runs" is optional (defaults to execution.runs, then 1).
+evaluations:
+  - generator: "nemotron-nano-30b"
+    judge: "mixtral-judge"
+    runs: 1
+
+  # Uncomment below to add more evaluations to the sweep.
+  # - generator: "generator-b"
+  #   judge: "mixtral-judge"
+  #   runs: 2
+  #
+  # - generator: "nemotron-super-49b"
+  #   judge: "mixtral-judge"
+  #   runs: 5
+
+execution:
+  top_k: 5
+  max_workers: 8
+  # timeout: 120    # per-request timeout in seconds (default: 120)
+  # min_coverage: 0.8  # abort if retrieval file covers < 80% of queries (default: 0.0 = no check)
+
+output:
+  results_dir: "data/eval"

nemo_retriever/pyproject.toml

@@ -110,6 +110,11 @@ benchmarks = [
   "open-clip-torch==3.2.0",
 ]
 
+eval = [
+  "litellm>=1.40.0",
+  "pyyaml>=6.0",
+  "tenacity>=8.0.0",
+]
 dev = [
   "build>=1.2.2",
   "pytest>=8.0.2",

nemo_retriever/src/nemo_retriever/adapters/cli/main.py

@@ -10,6 +10,7 @@
 from nemo_retriever.utils.benchmark import app as benchmark_app
 from nemo_retriever.chart import app as chart_app
 from nemo_retriever.utils.compare import app as compare_app
+from nemo_retriever.evaluation.cli import app as eval_app
 from nemo_retriever.harness import app as harness_app
 from nemo_retriever.html import __main__ as html_main
 from nemo_retriever.utils.image import app as image_app
@@ -28,6 +29,7 @@
 app.add_typer(local_app, name="local")
 app.add_typer(chart_app, name="chart")
 app.add_typer(compare_app, name="compare")
+app.add_typer(eval_app, name="eval")
 app.add_typer(benchmark_app, name="benchmark")
 app.add_typer(harness_app, name="harness")
 app.add_typer(vector_store_app, name="vector-store")

nemo_retriever/src/nemo_retriever/evaluation/__init__.py

@@ -0,0 +1,77 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-25, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""QA evaluation framework for nemo_retriever.
+
+Provides pluggable retrieval, generation, judging, and orchestration
+components for measuring LLM answer quality given retrieved context.
+
+The ``EvalOperator`` base class bridges ``graph.AbstractOperator`` into the
+evaluation domain, enabling ``>>`` chaining, ``Graph.execute()``, and
+executor compatibility for all evaluation operators.
+
+Types, scoring, and ``EvalOperator`` are always available.
+Modules that depend on ``litellm`` (generators, judges, generation,
+judging, orchestrator, config) are lazy-loaded so that lightweight
+consumers can use scoring without installing the ``[eval]`` extra::
+
+    pip install nemo-retriever[eval]
+"""
+
+from nemo_retriever.evaluation.eval_operator import EvalOperator
+from nemo_retriever.evaluation.scoring import score_dataframe
+from nemo_retriever.evaluation.types import (
+    AnswerJudge,
+    GenerationResult,
+    JudgeResult,
+    LLMClient,
+    RetrievalResult,
+    RetrieverStrategy,
+)
+
+_LAZY_IMPORTS = {
+    "QAGenerationOperator": "nemo_retriever.evaluation.generation",
+    "JudgingOperator": "nemo_retriever.evaluation.judging",
+    "ScoringOperator": "nemo_retriever.evaluation.scoring_operator",
+    "RetrievalLoaderOperator": "nemo_retriever.evaluation.retrieval_loader",
+    "LiteLLMClient": "nemo_retriever.evaluation.generators",
+    "LLMJudge": "nemo_retriever.evaluation.judges",
+    "QAEvalPipeline": "nemo_retriever.evaluation.orchestrator",
+    "load_eval_config": "nemo_retriever.evaluation.config",
+    "build_eval_chain": "nemo_retriever.evaluation.config",
+    "build_eval_pipeline": "nemo_retriever.evaluation.config",
+    "run_eval_sweep": "nemo_retriever.evaluation.runner",
+}
+
+
+def __getattr__(name: str):
+    if name in _LAZY_IMPORTS:
+        import importlib
+
+        module = importlib.import_module(_LAZY_IMPORTS[name])
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "AnswerJudge",
+    "EvalOperator",
+    "GenerationResult",
+    "JudgeResult",
+    "JudgingOperator",
+    "LLMClient",
+    "LLMJudge",
+    "LiteLLMClient",
+    "QAEvalPipeline",
+    "QAGenerationOperator",
+    "RetrievalLoaderOperator",
+    "RetrievalResult",
+    "RetrieverStrategy",
+    "ScoringOperator",
+    "build_eval_chain",
+    "build_eval_pipeline",
+    "load_eval_config",
+    "run_eval_sweep",
+    "score_dataframe",
+]