create_final_table.py

import os
import pandas as pd
import argparse
from pathlib import Path
import glob
import numpy as np

def parse_args():
    parser = argparse.ArgumentParser(description="Aggregate STAR-Bench results into a summary table by typology.")
    parser.add_argument("--input_dir", type=str, required=True, help="Root folder containing the results (OUTPUT_BASE_FOLDER).")
    parser.add_argument("--output_file", type=str, default="final_summary_table.csv", help="Path of the output CSV file.")
    parser.add_argument(
        "--compute-pareto", action="store_true",
        help="After the summary table is written, also compute the "
             "latency/accuracy/energy Pareto frontiers via "
             "starbench_performance_evaluation and dump them next to the "
             "output file (in a 'pareto/' subfolder).",
    )
    parser.add_argument(
        "--pareto-plot", action="store_true",
        help="When --compute-pareto is set, also render matplotlib scatter "
             "plots for every frontier (requires matplotlib).",
    )
    return parser.parse_args()

def extract_typology(file_path, base_dir):
    """
    Deduce the typology (e.g. OPT-OPT, Intra_blue-green) from the file path.
    Assumes the structure: base_dir / TYPOLOGY / DATASET / results_... / ...
    """
    try:
        path_obj = Path(file_path)
        base_obj = Path(base_dir)

        # Compute the relative path with respect to the base folder
        rel_path = path_obj.relative_to(base_obj)

        # The first part of the relative path should be the Typology
        # E.g.: Intra_blue-green / Agri / results_mi / ...
        parts = rel_path.parts
        if len(parts) >= 2:
            return parts[0] # Returns "Intra_blue-green"
        else:
            return "Unknown"
    except ValueError:
        return "Unknown"

def main():
    args = parse_args()
    root_dir = args.input_dir
    
    print(f"Scanning results in: {root_dir}")

    # Recursively search for all summary_report_overall.csv files
    search_pattern = os.path.join(root_dir, "**", "summary_report_overall.csv")
    all_files = glob.glob(search_pattern, recursive=True)

    if not all_files:
        print("No 'summary_report_overall.csv' file found.")
        return

    print(f"Found {len(all_files)} partial reports.")
    
    aggregated_data = []

    for file_path in all_files:
        try:
            # Read the partial CSV
            df = pd.read_csv(file_path)

            # Clean the data (convert N/A to NaN for correct computation)
            # Note: Added CPU and RAM columns here
            numeric_cols = [
                "Avg Final RMSE", "QAC (calc)", "Energy (calc)", 
                "Avg ProcTime(s)", "Avg Power(W)", "Avg CPU(%)", "Avg RAM(%)",
                "#Success", "#Total"
            ]
            
            for col in numeric_cols:
                if col in df.columns:
                    df[col] = pd.to_numeric(df[col], errors='coerce')

            # Determine the typology
            typology = extract_typology(file_path, root_dir)
            df['Typology'] = typology

            aggregated_data.append(df)

        except Exception as e:
            print(f"Error reading {file_path}: {e}")

    if not aggregated_data:
        print("No valid data extracted.")
        return

    # Merge everything into a single DataFrame
    full_df = pd.concat(aggregated_data, ignore_index=True)

    # --- AGGREGATION ---
    print("Computing averages by typology...")

    # Columns to aggregate
    agg_dict = {
        'Avg Final RMSE': 'mean',   # RMSE requested
        'QAC (calc)': 'mean',       # QAC requested
        'Energy (calc)': 'mean',    # Energy requested
        'Avg Power(W)': 'mean',     # Power requested
        'Avg CPU(%)': 'mean',       # CPU requested
        'Avg RAM(%)': 'mean',       # RAM requested
        'Avg ProcTime(s)': 'mean',  # Processing Time (All Time excluded)
        '#Success': 'sum',
        '#Total': 'sum'
    }
    
    # Filter out columns that might be missing to avoid KeyError
    existing_agg_cols = {k: v for k, v in agg_dict.items() if k in full_df.columns}
    
    summary_df = full_df.groupby(['Typology', 'Pipeline']).agg(existing_agg_cols).reset_index()

    # Rename columns for final readability
    rename_map = {
        'Avg Final RMSE': 'Mean RMSE',
        'QAC (calc)': 'Mean QAC',
        'Energy (calc)': 'Mean Energy (J)',
        'Avg Power(W)': 'Mean Power (W)',
        'Avg CPU(%)': 'Mean CPU (%)',
        'Avg RAM(%)': 'Mean RAM (%)',
        'Avg ProcTime(s)': 'Mean Proc Time (s)',
        '#Success': 'Total Successes',
        '#Total': 'Total Runs'
    }
    summary_df.rename(columns=rename_map, inplace=True)

    # Compute global success rate by typology
    if 'Total Successes' in summary_df.columns and 'Total Runs' in summary_df.columns:
        summary_df['Success Rate (%)'] = (summary_df['Total Successes'] / summary_df['Total Runs']) * 100

    # Sort: first by Typology, then by decreasing QAC
    sort_cols = ['Typology']
    if 'Mean QAC' in summary_df.columns:
        sort_cols.append('Mean QAC')
        summary_df = summary_df.sort_values(by=sort_cols, ascending=[True, False])
    else:
        summary_df = summary_df.sort_values(by='Typology')

    # Save
    output_path = args.output_file
    # Format floats nicely
    summary_df.to_csv(output_path, index=False, float_format='%.4f')

    print(f"\n=== DONE ===")
    print(f"Summary table saved to: {output_path}")

    # Console preview (key columns)
    preview_cols = ['Typology', 'Pipeline', 'Mean RMSE', 'Mean QAC', 'Mean Energy (J)', 'Mean CPU (%)', 'Mean RAM (%)']
    available_preview = [c for c in preview_cols if c in summary_df.columns]
    print("\nResults preview (Top 2 per typology):")
    print(summary_df.groupby('Typology').head(2)[available_preview].to_string())

    # --- Optional Pareto analysis --------------------------------------
    if args.compute_pareto:
        try:
            import starbench_performance_evaluation as perf_eval
        except ImportError as e:
            print(f"[warn] Could not import starbench_performance_evaluation: {e}")
            return

        pareto_dir = Path(output_path).with_suffix('').parent / "pareto"
        print(f"\nComputing Pareto frontiers in {pareto_dir} ...")
        try:
            df = perf_eval.load_summary(root_dir)
            frontiers = perf_eval.compute_all_frontiers(df)
            perf_eval.write_frontiers(frontiers, pareto_dir)
            if args.pareto_plot:
                perf_eval.plot_frontiers(frontiers, pareto_dir)
            print(f"Pareto analysis complete: {pareto_dir}")
        except Exception as e:
            print(f"[error] Pareto computation failed: {e}")

if __name__ == "__main__":
    main()