[TTS] Updated TTS comparison report

Author: artem-gorodetskii

scripts/tts_comparison_report/reporting/components/boxplots.py

@@ -50,6 +50,8 @@ class BoxPlotsConfig:
     outlier_markersize: float = 3.0
     outlier_alpha: float = 0.5
 
+    unavailable_text_color: str = "#CD5C5C"
+
 
 def _style_boxplot(
     bp: dict[str, PathPatch],
@@ -98,6 +100,42 @@ def _add_mean_ci_labels(
         ax.text(x + x_offset, mean + y_offset, label, ha="left", va="center", fontsize=cfg.fontsize)
 
 
+def _mean_exceeds_plot_range(
+    baseline: np.ndarray,
+    candidate: np.ndarray,
+    metric: DistributionMetricSpec,
+) -> bool:
+    if metric.plot_range is None:
+        return False
+
+    upper_limit = metric.plot_range[1]
+    return bool(baseline.mean() > upper_limit or candidate.mean() > upper_limit)
+
+
+def _render_plot_not_shown(
+    ax: Axes,
+    metric: DistributionMetricSpec,
+    cfg: BoxPlotsConfig,
+) -> None:
+    if metric.plot_range is None:
+        raise ValueError(f"Metric '{metric.report_name}' does not define a plot range.")
+
+    upper_limit = metric.plot_range[1]
+
+    ax.set_title(metric.report_name, fontsize=cfg.fontsize_title)
+    ax.set_axis_off()
+    ax.text(
+        0.5,
+        0.5,
+        f"Plot not shown.\nMean {metric.report_name} exceeds {upper_limit:.0%} display limit.",
+        ha="center",
+        va="center",
+        color=cfg.unavailable_text_color,
+        fontsize=cfg.fontsize_title,
+        transform=ax.transAxes,
+    )
+
+
 def _configure_boxplot_axis(
     ax: Axes,
     metric: DistributionMetricSpec,
@@ -170,6 +208,10 @@ def prepare_boxplots(
             ax = axs[plot_idx]
             plot_idx += 1
 
+            if _mean_exceeds_plot_range(baseline, candidate, metric):
+                _render_plot_not_shown(ax, metric, cfg)
+                continue
+
             bp = ax.boxplot(
                 [baseline, candidate],
                 positions=[1, 2],

scripts/tts_comparison_report/reporting/constants.py

@@ -51,6 +51,9 @@
 # Number of decimal digits used when formatting p-values in statistical tests.
 P_VAL_ROUND_DIGITS: int = 4
 
+# Default signature version used to sign S3 client requests.
+S3_SIGNATURE_VERSION: str = "s3"
+
 # Default lifetime of generated S3 presigned links in seconds (one year).
 S3_LINK_EXPIRES_IN: int = 31536000
 

scripts/tts_comparison_report/reporting/models.py

@@ -12,17 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import hashlib
+import math
 from dataclasses import dataclass, field
 from enum import Enum
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Optional, Self
 
-from scripts.tts_comparison_report.reporting.constants import TQDM_NCOLS
+from scripts.tts_comparison_report.reporting.constants import BENCHMARK_META, TQDM_NCOLS
 from scripts.tts_comparison_report.reporting.storage import BaseStorage
 from tqdm import tqdm
 
-
 _REQUIRED_SAMPLE_ID_KEYS: list[str] = [
     "pred_audio_filepath",
     "gt_text",
@@ -309,6 +309,8 @@ def from_storage(
 
         Raises:
             FileNotFoundError: If the expected results directory is missing.
+            ValueError: If a recognized benchmark directory does not use the required
+                '<configuration>_<language>_<benchmark>' naming format.
         """
         obj = cls(name=bucket_name, path=bucket_path)
         results_path = bucket_path / bucket_structure.eval_output_subdir
@@ -334,8 +336,23 @@ def from_storage(
                 storage=storage,
             )
             if obj.configuration_str is None:
-                suffix = f"_{name}"
-                obj.configuration_str = dir_name[: -len(suffix)]
+                lang = BENCHMARK_META[name]
+                suffix = f"_{lang}_{name}"
+
+                if not dir_name.endswith(suffix):
+                    raise ValueError(
+                        f"Unsupported results directory name '{dir_name}' for benchmark '{name}': "
+                        f"expected '<configuration>{suffix}'."
+                    )
+
+                configuration_str = dir_name[: -len(suffix)]
+
+                if not configuration_str:
+                    raise ValueError(
+                        f"Missing configuration prefix in results directory '{dir_name}' for benchmark '{name}': "
+                        f"expected '<configuration>{suffix}'."
+                    )
+                obj.configuration_str = configuration_str
 
         return obj
 
@@ -391,11 +408,19 @@ def get_metric_avg_value(
         if metric_name not in metrics:
             return None
 
+        value = metrics[metric_name]
+
+        if value is None:
+            return None
+
         value = _validate_numeric_metric_value(
-            value=metrics[metric_name],
+            value=value,
             metric_name=metric_name,
             context=f"averaged metrics for benchmark '{benchmark_name}'",
         )
+        if math.isnan(value):
+            return None
+
         return value
 
     def _get_metric_stats(
@@ -423,6 +448,11 @@ def _get_metric_stats(
                 metric_name=metric_name,
                 context=validation_context,
             )
+            if math.isnan(value):
+                raise ValueError(
+                    f"Metric '{metric_name}' in {validation_context} contains NaN; "
+                    "statistical tests and box plots require non-NaN samples."
+                )
             output.append(value)
 
         if not output:

scripts/tts_comparison_report/reporting/s3_client.py

@@ -17,6 +17,8 @@
 import boto3
 from botocore.config import Config
 
+from scripts.tts_comparison_report.reporting.constants import S3_SIGNATURE_VERSION
+
 
 @dataclass
 class S3Config:
@@ -26,6 +28,7 @@ class S3Config:
     endpoint_url: str
     region_name: str
     connect_timeout: int = 10
+    signature_version: str = S3_SIGNATURE_VERSION
 
 
 class S3Client:
@@ -38,13 +41,17 @@ def __init__(
         aws_secret_access_key: str,
     ) -> None:
         self.cfg = cfg
+        config = Config(
+            connect_timeout=cfg.connect_timeout,
+            signature_version=cfg.signature_version,
+        )
         self.client = boto3.client(
             "s3",
             endpoint_url=cfg.endpoint_url,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
             region_name=cfg.region_name,
-            config=Config(connect_timeout=cfg.connect_timeout),
+            config=config,
         )
 
     def upload_fileobj(

scripts/tts_comparison_report/templates/audio_report.jinja

@@ -65,7 +65,38 @@
 
     .sidebar-left {
       grid-column: 1;
-      padding-right: 20px;
+      min-width: 0;
+      height: var(--sidebar-height);
+      padding-right: 12px;
+      max-height: calc(100vh - 64px);
+
+      display: flex;
+      flex-direction: column;
+      overflow: hidden;
+    }
+
+    .sidebar-left > h2 {
+      flex: 0 0 auto;
+      align-self: flex-start;
+    }
+
+    .sidebar-left > ul {
+      flex: 1 1 auto;
+      min-height: 0;
+      padding-bottom: 64px;
+      overflow-y: auto;
+      overflow-x: hidden;
+      overscroll-behavior: contain;
+
+      /* Hide scrollbar */
+      /* Firefox and old Microsoft browsers */
+      scrollbar-width: none;
+      -ms-overflow-style: none;
+    }
+
+    /* Chrome, Edge, and Safari */
+    .sidebar-left > ul::-webkit-scrollbar {
+      display: none;
     }
 
     .sidebar-right {
@@ -112,8 +143,8 @@
 
     .sidebar ul ul {
       margin-top: 8px;
-      margin-left: 14px;
-      padding-left: 12px;
+      margin-left: 6px;
+      padding-left: 8px;
       border-left: 1px solid var(--border);
     }
 
@@ -137,6 +168,24 @@
       margin-bottom: 1.5rem;
     }
 
+    .benchmark-link {
+      position: relative;
+      display: block;
+      max-width: 100%;
+      padding-left: 14px;
+      overflow-wrap: anywhere;
+      word-break: break-word;
+    }
+
+    .benchmark-link::before {
+      content: "›";
+      position: absolute;
+      left: 0;
+      top: 0;
+      color: currentColor;
+      font-weight: 700;
+    }
+
     .link-comment {
       color: var(--highlighted);
       font-size: 1rem;
@@ -264,7 +313,7 @@
       <h2>Content</h2>
         <ul>
           {% for section_id, section_name in benchmark_section_info %}
-            <li><a href="#{{ section_id }}">{{ section_name }}</a></li>
+            <li><a class="benchmark-link" href="#{{ section_id }}">{{ section_name }}</a></li>
           {% endfor %}
         </ul>
     </aside>
@@ -322,16 +371,20 @@
 <script>
   const ACTIVE_SECTION_OFFSET = 140;
   const BOTTOM_THRESHOLD = 20;
+  const SIDEBAR_STICKY_TOP = 32;
 
   const sections = [...document.querySelectorAll("h2[id], h3[id]")];
   const navLinks = [...document.querySelectorAll('.sidebar-left a[href^="#"]')];
+  let clickedNavLink = null;
 
   const linkMap = new Map();
   navLinks.forEach(link => {
     const href = link.getAttribute("href");
     linkMap.set(href.slice(1), link);
 
     link.addEventListener("click", () => {
+      clickedNavLink = link;
+
       navLinks.forEach(l => l.classList.remove("active"));
       link.classList.add("active");
     });
@@ -358,12 +411,41 @@
       const activeLink = linkMap.get(currentSection);
       if (activeLink) {
         activeLink.classList.add("active");
+
+        if (activeLink !== clickedNavLink) {
+          clickedNavLink = null;
+          activeLink.scrollIntoView({
+            block: "nearest",
+            inline: "nearest",
+            behavior: "auto",
+          });
+        }
       }
     }
   }
 
+  const sidebar = document.querySelector(".sidebar-left");
+
+  function updateSidebarHeight() {
+    if (!sidebar) return;
+
+    const top = Math.max(
+      sidebar.getBoundingClientRect().top,
+      SIDEBAR_STICKY_TOP
+    );
+    sidebar.style.setProperty(
+      "--sidebar-height",
+      `${Math.max(0, window.innerHeight - top)}px`
+    );
+  }
+
   window.addEventListener("scroll", updateActiveLink, { passive: true });
+  window.addEventListener("scroll", updateSidebarHeight, { passive: true });
+  window.addEventListener("resize", updateSidebarHeight);
   window.addEventListener("load", updateActiveLink);
+  window.addEventListener("load", updateSidebarHeight);
+
+  updateSidebarHeight();
 </script>
 
 </body>