eval_dataset.py

#!/usr/bin/env python3
"""TTS Dataset Evaluation Tool

Evaluates TTS synthesized audio using ASR models and computes Kana-CER/CER metrics.
Designed for Common Kanji dataset with tagged pronunciation spans.

Usage:
    python3 eval_dataset.py \\
        --dataset-dir path/to/dataset \\
        --source-jsonl path/to/ground_truth.jsonl
"""

import argparse
import sys
from pathlib import Path

from eval.pipeline import EvaluationPipeline


def main():
    parser = argparse.ArgumentParser(
        description='Evaluate TTS synthesized audio using ASR models',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  # Basic evaluation (dataset auto-downloads)
  python3 eval_dataset.py \\
      --dataset-dir data/common_kanji_test

  # With custom models
  python3 eval_dataset.py \\
      --dataset-dir data/common_kanji_test \\
      --kana-model ./checkpoint-10000 \\
      --text-model openai/whisper-large-v3-turbo
        """
    )

    # Required arguments
    parser.add_argument(
        '--dataset-dir',
        required=True,
        type=Path,
        help='Dataset directory containing synthesized_audio/ subdirectory'
    )
    parser.add_argument(
        '--source-jsonl',
        type=Path,
        default=Path('data/common_kanji_source.jsonl'),
        help='Source JSONL file with ground truth (default: data/common_kanji_source.jsonl, auto-downloads if missing)'
    )

    # Optional arguments
    parser.add_argument(
        '--kana-model',
        default='sbintuitions/kana-whisper',
        help='ASR model for Kana-CER computation (default: sbintuitions/kana-whisper)'
    )
    parser.add_argument(
        '--text-model',
        default='openai/whisper-large-v3-turbo',
        help='ASR model for Standard CER computation (default: whisper-large-v3-turbo)'
    )
    parser.add_argument(
        '--result-dir',
        type=Path,
        help='Output directory (default: dataset-dir/eval_results)'
    )
    parser.add_argument(
        '--skip-asr',
        action='store_true',
        help='Skip ASR transcription if transcription files already exist'
    )

    args = parser.parse_args()

    # Create and run pipeline
    try:
        pipeline = EvaluationPipeline(
            dataset_dir=args.dataset_dir,
            source_jsonl=args.source_jsonl,
            kana_model=args.kana_model,
            text_model=args.text_model,
            result_dir=args.result_dir,
            skip_existing=args.skip_asr
        )

        pipeline.run()
        return 0

    except Exception as e:
        print(f"Error: {e}", file=sys.stderr)
        return 1


if __name__ == '__main__':
    sys.exit(main())