Implement Prosody metrics for TTS evaluation (#16031)

* Support TTS speaker id format on SpeakerFilter class

Signed-off-by: Edresson Casanova <edresson1@gmail.com>

* Add prosody evaluation metrics

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Clean up emotion encoder CLI

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Simplify prosody metric defaults

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Use default prosody emotion embedding

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Remove emotion encoder CLI

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Fix black formatting

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Update speaker filter test expectation

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Add generated-to-ground-truth audio CER metric

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Add generated-to-ground-truth audio WER metric

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

* Move prosody metric defaults into entrypoint

Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

---------

Signed-off-by: Edresson Casanova <edresson1@gmail.com>
Signed-off-by: Edresson Casanova <ecasanova@nvidia.com>

Author: Edresson

examples/tts/magpietts_inference.py

@@ -167,9 +167,13 @@ def run_inference_and_evaluation(
     # CSV headers
     csv_header = (
         "checkpoint_name,dataset,cer_filewise_avg,wer_filewise_avg,cer_cumulative,"
-        "wer_cumulative,ssim_pred_gt_avg,ssim_pred_context_avg,ssim_gt_context_avg,"
+        "wer_cumulative,cer_pred_gt_audio_filewise_avg,cer_pred_gt_audio_cumulative,"
+        "wer_pred_gt_audio_filewise_avg,wer_pred_gt_audio_cumulative,"
+        "ssim_pred_gt_avg,ssim_pred_context_avg,ssim_gt_context_avg,"
         "ssim_pred_gt_avg_alternate,ssim_pred_context_avg_alternate,"
-        "ssim_gt_context_avg_alternate,cer_gt_audio_cumulative,wer_gt_audio_cumulative,"
+        "ssim_gt_context_avg_alternate,esim_pred_gt_avg,ems_pred_gt_avg,"
+        "pitch_distance_avg,intensity_distance_avg,speech_rate_distance_avg,"
+        "cer_gt_audio_cumulative,wer_gt_audio_cumulative,"
         "utmosv2_avg,total_gen_audio_seconds,frechet_codec_distance,"
         "eou_cutoff_rate,eou_silence_rate,eou_noise_rate,eou_error_rate,"
         "katakana_cer_filewise_avg,katakana_cer_cumulative"
@@ -303,6 +307,8 @@ def run_inference_and_evaluation(
                 with_utmosv2=eval_config.with_utmosv2,
                 with_fcd=eval_config.with_fcd,
                 codec_model_path=eval_config.codec_model_path,
+                with_prosody_metrics=eval_config.with_prosody_metrics,
+                prosody_model_size=eval_config.prosody_model_size,
                 strip_text_annotations_for_metrics=eval_config.strip_text_annotations_for_metrics,
                 device=eval_config.device,
                 asr_batch_size=eval_config.asr_batch_size,
@@ -456,6 +462,8 @@ def main(argv=None):
         with_utmosv2=not args.disable_utmosv2,
         with_fcd=not args.disable_fcd,
         codec_model_path=args.codecmodel_path if not args.disable_fcd else None,
+        with_prosody_metrics=args.with_prosody_metrics,
+        prosody_model_size=args.prosody_model_size,
         strip_text_annotations_for_metrics=args.strip_text_annotations_for_metrics,
         asr_batch_size=args.asr_batch_size,
         eou_batch_size=args.eou_batch_size,

nemo/collections/common/data/lhotse/sampling.py

@@ -342,6 +342,11 @@ def __call__(self, example) -> bool:
                 else:
                     speaker_id = getattr(supervision, field, None)
 
+                # Support the TTS speaker ID format:
+                # | Language:en Dataset:<dataset_name> Speaker:<speaker_id> |
+                if isinstance(speaker_id, str) and "Speaker:" in speaker_id:
+                    speaker_id = speaker_id.rsplit("Speaker:", maxsplit=1)[-1].split("|", maxsplit=1)[0].strip()
+
                 if speaker_id in excluded_speaker_ids:
                     return False
         return True