Dashboard cleanup: correct retrieval numbers, wire test-test/test-train hexbins, drop unused panels

Author: rukubrakov

results/calibration_analysis_bs2048_v2_step44k.png

@@ -0,0 +1,2 @@
+split	model	n	hit@1	hit@5	hit@20
+test	msg_exact_mces_1020_no_meta_own_val_weights_bs2048_v2	17556	0.04665071770334928	0.12593984962406016	0.265436318067897

results/simba_retrieval_bs2048_v2_step44k_fixed_all.tsv

@@ -16,19 +16,6 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
-from rdkit import Chem
-from rdkit.Chem import Descriptors
-from tqdm.auto import tqdm
-
-
-def _exact_mass(smi: str) -> float:
-    mol = Chem.MolFromSmiles(smi)
-    return Descriptors.ExactMolWt(mol) if mol else float("nan")
-
-
-def _heavy_atoms(smi: str) -> int:
-    mol = Chem.MolFromSmiles(smi)
-    return mol.GetNumHeavyAtoms() if mol else 0
 
 
 def main():
@@ -39,7 +26,6 @@ def main():
 
     out = Path(args.output)
     out.parent.mkdir(parents=True, exist_ok=True)
-    fig2_path = out.with_name(out.stem + "_pop.png")
 
     print("Loading CSV ...")
     rows = []
@@ -53,7 +39,6 @@ def main():
     simba_pred = np.array([float(r["simba_pred_mces"]) for r in rows])
     simba_gt = np.array([float(r["simba_gt_mces"]) for r in rows])
     oracle_pred = np.array([float(r["oracle_pred_mces"]) for r in rows])
-    oracle_gt = np.array([float(r["oracle_gt_mces"]) for r in rows])
     simba_tani = np.array([float(r["simba_tanimoto"]) for r in rows])
     covered = np.array([r["covered"] == "True" for r in rows])
 
@@ -71,22 +56,6 @@ def main():
         cosine_oracle = 1.0 - oracle_pred / 40.0
         cosine_label = "Cosine sim (embedding, 1−pred/40) ← re-run diagnose_retrieval to get spectral"
 
-    # ── RDKit properties (cached) ─────────────────────────────────────────────
-    print("Computing molecular properties ...")
-    mass_cache: dict[str, float] = {}
-    atom_cache: dict[str, int] = {}
-    unique_smi = {r["test_smi"] for r in rows} | {r["simba_smi"] for r in rows}
-    for s in tqdm(unique_smi, desc="RDKit"):
-        mass_cache[s] = _exact_mass(s)
-        atom_cache[s] = _heavy_atoms(s)
-
-    mass_test = np.array([mass_cache[r["test_smi"]] for r in rows])
-    mass_simba = np.array([mass_cache[r["simba_smi"]] for r in rows])
-    atoms_test = np.array([atom_cache[r["test_smi"]] for r in rows])
-    atoms_simba = np.array([atom_cache[r["simba_smi"]] for r in rows])
-    mass_diff = np.abs(mass_test - mass_simba)
-    atoms_diff = np.abs(atoms_test - atoms_simba).astype(float)
-
     om = covered  # covered mask
 
     # ── Print summary stats ───────────────────────────────────────────────────
@@ -97,30 +66,13 @@ def main():
     print(
         f"  {'spectral' if has_spectral else 'embedding'} cosine oracle pick: mean={cosine_oracle[om].mean():.3f}  median={np.median(cosine_oracle[om]):.3f}"
     )
-    print(
-        f"  mass_diff pair:         mean={mass_diff[om].mean():.1f} Da  median={np.median(mass_diff[om]):.1f} Da"
-    )
-    print(
-        f"  atoms_diff pair:        mean={atoms_diff[om].mean():.1f}   median={np.median(atoms_diff[om]):.1f}"
-    )
     print(f"  Tanimoto SIMBA pair:    mean={simba_tani[om].mean():.3f}")
 
-    HIGH = om & (simba_err > 25)
-    LOW = om & (simba_err < 5)
-    print(
-        f"\n  High error (>25): n={HIGH.sum():,}  mass_diff mean={mass_diff[HIGH].mean():.1f} Da  "
-        f"atoms_diff mean={atoms_diff[HIGH].mean():.1f}  tani mean={simba_tani[HIGH].mean():.3f}"
-    )
-    print(
-        f"  Low error  (<5):  n={LOW.sum():,}   mass_diff mean={mass_diff[LOW].mean():.1f} Da  "
-        f"atoms_diff mean={atoms_diff[LOW].mean():.1f}  tani mean={simba_tani[LOW].mean():.3f}"
-    )
-
-    # ── Figure 1: Error anatomy (2×3) ─────────────────────────────────────────
-    fig, axes = plt.subplots(2, 3, figsize=(18, 11))
+    # ── Figure 1: panels 1 + 6 ────────────────────────────────────────────────
+    fig, axes = plt.subplots(1, 2, figsize=(14, 5))
 
     # 1. Cosine sim distribution: SIMBA pick vs oracle pick
-    ax = axes[0, 0]
+    ax = axes[0]
     bins1 = np.linspace(0, 1, 51)
     ax.hist(
         cosine_oracle[om],
@@ -144,50 +96,8 @@ def main():
     ax.legend(fontsize=9)
     ax.grid(True, alpha=0.2)
 
-    # 2. simba_err vs mass_diff (hexbin)
-    ax = axes[0, 1]
-    hb = ax.hexbin(mass_diff[om], simba_err[om], gridsize=30, cmap="Reds", mincnt=1)
-    plt.colorbar(hb, ax=ax, label="count")
-    ax.axhline(0, color="k", lw=1, ls="--")
-    ax.set_xlabel("|mass_test − mass_SIMBA_pick|  (Da)")
-    ax.set_ylabel("simba_err  (GT MCES − pred MCES)")
-    ax.set_title("2 · Calibration error vs mass difference", fontweight="bold")
-    ax.grid(True, alpha=0.2)
-
-    # 3. simba_err vs atoms_diff
-    ax = axes[0, 2]
-    hb = ax.hexbin(atoms_diff[om], simba_err[om], gridsize=30, cmap="Purples", mincnt=1)
-    plt.colorbar(hb, ax=ax, label="count")
-    ax.axhline(0, color="k", lw=1, ls="--")
-    ax.set_xlabel("|atoms_test − atoms_SIMBA_pick|")
-    ax.set_ylabel("simba_err")
-    ax.set_title("3 · Calibration error vs atom count diff", fontweight="bold")
-    ax.grid(True, alpha=0.2)
-
-    # 4. simba_err vs atoms_test
-    ax = axes[1, 0]
-    hb = ax.hexbin(
-        atoms_test[om].astype(float), simba_err[om], gridsize=30, cmap="Blues", mincnt=1
-    )
-    plt.colorbar(hb, ax=ax, label="count")
-    ax.axhline(0, color="k", lw=1, ls="--")
-    ax.set_xlabel("Heavy atom count of test molecule")
-    ax.set_ylabel("simba_err")
-    ax.set_title("4 · Calibration error vs test mol size", fontweight="bold")
-    ax.grid(True, alpha=0.2)
-
-    # 5. simba_err vs Tanimoto(test, SIMBA pick)
-    ax = axes[1, 1]
-    hb = ax.hexbin(simba_tani[om], simba_err[om], gridsize=30, cmap="Oranges", mincnt=1)
-    plt.colorbar(hb, ax=ax, label="count")
-    ax.axhline(0, color="k", lw=1, ls="--")
-    ax.set_xlabel("Tanimoto similarity (test, SIMBA pick)")
-    ax.set_ylabel("simba_err")
-    ax.set_title("5 · Calibration error vs structural similarity", fontweight="bold")
-    ax.grid(True, alpha=0.2)
-
     # 6. GT MCES distribution of SIMBA picks with pred mean
-    ax = axes[1, 2]
+    ax = axes[1]
     bins6 = np.arange(0, 42.5, 2.5)
     ax.hist(simba_gt[om], bins=bins6, color="#5B8DB8", edgecolor="none", alpha=0.85)
     ax.axvline(
@@ -218,113 +128,7 @@ def main():
     )
     plt.tight_layout()
     plt.savefig(out, dpi=140, bbox_inches="tight")
-    print(f"\nFigure 1 saved → {out}")
-
-    # ── Figure 2: Population analysis (2×2) ──────────────────────────────────
-    fig2, axes2 = plt.subplots(2, 2, figsize=(14, 11))
-
-    # A. mass_diff distribution: high vs low error
-    ax = axes2[0, 0]
-    bins_m = np.linspace(0, np.percentile(mass_diff[om], 99), 60)
-    ax.hist(
-        mass_diff[LOW],
-        bins=bins_m,
-        color="#4E9A7A",
-        alpha=0.75,
-        density=True,
-        edgecolor="none",
-        label=f"err<5   n={LOW.sum():,}  μ={mass_diff[LOW].mean():.1f} Da",
-    )
-    ax.hist(
-        mass_diff[HIGH],
-        bins=bins_m,
-        color="#E07B54",
-        alpha=0.75,
-        density=True,
-        edgecolor="none",
-        label=f"err>25  n={HIGH.sum():,}  μ={mass_diff[HIGH].mean():.1f} Da",
-    )
-    ax.set_xlabel("|mass_test − mass_SIMBA_pick|  (Da)")
-    ax.set_ylabel("density")
-    ax.set_title("A · Mass diff: low-error vs high-error pairs", fontweight="bold")
-    ax.legend(fontsize=9)
-    ax.grid(True, alpha=0.2)
-
-    # B. 2D hexbin (mass_test, mass_simba) colored by mean simba_err
-    ax = axes2[0, 1]
-    lim = np.percentile(np.concatenate([mass_test[om], mass_simba[om]]), 99)
-    hb = ax.hexbin(
-        mass_test[om],
-        mass_simba[om],
-        C=simba_err[om],
-        gridsize=35,
-        cmap="RdYlGn_r",
-        reduce_C_function=np.mean,
-        mincnt=3,
-    )
-    plt.colorbar(hb, ax=ax, label="mean simba_err")
-    ax.plot([0, lim], [0, lim], "k--", lw=1.2, alpha=0.7, label="test = SIMBA pick")
-    ax.set_xlim(0, lim)
-    ax.set_ylim(0, lim)
-    ax.set_xlabel("Exact mass — test molecule  (Da)")
-    ax.set_ylabel("Exact mass — SIMBA pick  (Da)")
-    ax.set_title("B · Mass pairs colored by mean calibration error", fontweight="bold")
-    ax.legend(fontsize=8)
-    ax.grid(True, alpha=0.2)
-
-    # C. Tanimoto distribution: high vs low error
-    ax = axes2[1, 0]
-    bins_t = np.linspace(0, 1, 41)
-    ax.hist(
-        simba_tani[LOW],
-        bins=bins_t,
-        color="#4E9A7A",
-        alpha=0.75,
-        density=True,
-        edgecolor="none",
-        label=f"err<5   μ={simba_tani[LOW].mean():.3f}",
-    )
-    ax.hist(
-        simba_tani[HIGH],
-        bins=bins_t,
-        color="#E07B54",
-        alpha=0.75,
-        density=True,
-        edgecolor="none",
-        label=f"err>25  μ={simba_tani[HIGH].mean():.3f}",
-    )
-    ax.set_xlabel("Tanimoto similarity (test, SIMBA pick)")
-    ax.set_ylabel("density")
-    ax.set_title("C · Tanimoto: low-error vs high-error pairs", fontweight="bold")
-    ax.legend(fontsize=9)
-    ax.grid(True, alpha=0.2)
-
-    # D. SIMBA pick GT vs oracle GT colored by simba_err
-    ax = axes2[1, 1]
-    hb = ax.hexbin(
-        oracle_gt[om],
-        simba_gt[om],
-        C=simba_err[om],
-        gridsize=30,
-        cmap="RdYlGn_r",
-        reduce_C_function=np.mean,
-        mincnt=3,
-    )
-    plt.colorbar(hb, ax=ax, label="mean simba_err")
-    ax.plot([0, 40], [0, 40], "k--", lw=1.2, alpha=0.7, label="SIMBA = oracle")
-    ax.set_xlabel("Oracle GT MCES")
-    ax.set_ylabel("SIMBA pick GT MCES")
-    ax.set_title("D · SIMBA GT vs oracle GT (colored by error)", fontweight="bold")
-    ax.legend(fontsize=8)
-    ax.grid(True, alpha=0.2)
-
-    fig2.suptitle(
-        "Population Analysis: High-error (err>25) vs Low-error (err<5) SIMBA Retrieval Pairs",
-        fontsize=12,
-    )
-    plt.tight_layout()
-    plt.savefig(fig2_path, dpi=140, bbox_inches="tight")
-    print(f"Figure 2 saved → {fig2_path}")
+    print(f"\nSaved → {out}")
 
 
 if __name__ == "__main__":

tools/analyze_calibration.py

@@ -392,18 +392,19 @@ def show_hexbin(path: Path | None, caption: str = ""):
                         show(_p, _p.stem)
 
     _ret8104 = Path(
-        "/home/nkubrakov/simba/results/simba_retrieval_bs2048_v2_step44k.tsv"
+        "/home/nkubrakov/simba/results/simba_retrieval_bs2048_v2_step44k_fixed_all.tsv"
     )
     if _ret8104.exists():
-        st.markdown("**Retrieval benchmark · step 44k (all 194k train spectra)**")
+        st.markdown("**Retrieval benchmark · step 44k · fixed preprocessing**")
         _df = pd.read_csv(_ret8104, sep="\t")
         _row = _df.iloc[0]
         _c1, _c2, _c3 = st.columns(3)
         _c1.metric("SIMBA hit@1", f"{_row['hit@1'] * 100:.2f}%")
         _c2.metric("SIMBA hit@5", f"{_row['hit@5'] * 100:.2f}%")
         _c3.metric("SIMBA hit@20", f"{_row['hit@20'] * 100:.2f}%")
         st.caption(
-            f"n={int(_row['n'])} · cosine-NN → Morgan FP transfer → Tanimoto ranking"
+            f"n={int(_row['n'])} · cosine-NN → Morgan FP transfer → Tanimoto ranking  "
+            f"(n_layers=5 · top-N intensity · sqrt+L2 norm)"
         )
 
     _cos_hex = Path(
@@ -460,27 +461,40 @@ def show_hexbin(path: Path | None, caption: str = ""):
         )
         show(_diag8104)
     if _diag8104_mol.exists():
-        st.markdown(
-            "**Molecular property analysis · step 44k** — Tanimoto, SIMBA GT vs oracle GT"
-        )
-        show(_diag8104_mol)
+        st.markdown("**Tanimoto analysis · step 44k** — oracle vs SIMBA pick")
+        img_mol = Image.open(_diag8104_mol)
+        w_mol, h_mol = img_mol.size
+        cropped_mol = img_mol.crop((0, 0, w_mol // 3, h_mol))
+        buf_mol = io.BytesIO()
+        cropped_mol.save(buf_mol, format="PNG")
+        st.image(buf_mol.getvalue(), use_container_width=False, width=500)
 
     _cal = Path(
         "/home/nkubrakov/simba/results/calibration_analysis_bs2048_v2_step44k.png"
     )
-    _cal_pop = Path(
-        "/home/nkubrakov/simba/results/calibration_analysis_bs2048_v2_step44k_pop.png"
-    )
     if _cal.exists():
         st.markdown(
-            "**Calibration error anatomy · step 44k** — cosine sim distributions, error vs mass/atoms/Tanimoto"
+            "**Calibration error anatomy · step 44k** — spectral cosine distributions + GT MCES of picks"
         )
         show(_cal)
-    if _cal_pop.exists():
+
+    _tt_hex = Path(
+        "/home/nkubrakov/simba/results/test_test_mces_hexbin_bs2048_v2_step44k_fixed_all.png"
+    )
+    if _tt_hex.exists():
+        st.markdown(
+            "**Test-test hexbin · step 44k** — GT-balanced (rho=0.574) vs unbalanced (rho=0.692)"
+        )
+        show(_tt_hex)
+
+    _ttr_hex = Path(
+        "/home/nkubrakov/simba/results/test_train_mces_hexbin_bs2048_v2_step44k_fixed_all.png"
+    )
+    if _ttr_hex.exists():
         st.markdown(
-            "**Calibration error population analysis** — high-error vs low-error pair properties"
+            "**Test-train hexbin · step 44k** — 5-panel cross-split analysis (rho=0.680)"
         )
-        show(_cal_pop)
+        show(_ttr_hex)
 
 st.markdown("---")
 

tools/dashboard/app.py

@@ -628,11 +628,8 @@ def get_fp(smi):
     plt.savefig(out_path, dpi=140, bbox_inches="tight")
     print(f"Figure 1 saved → {out_path}")
 
-    # ── Figure 2: molecular properties ───────────────────────────────────────
-    fig2, axes2 = plt.subplots(1, 3, figsize=(16, 5))
-
-    # 5. Tanimoto distributions
-    ax = axes2[0]
+    # ── Figure 2: Tanimoto distributions (panel 5) ───────────────────────────
+    fig2, ax = plt.subplots(1, 1, figsize=(6, 5))
     bins5 = np.linspace(0, 1, 41)
     ax.hist(
         oracle_tanis[om],
@@ -652,40 +649,11 @@ def get_fp(smi):
     )
     ax.set_xlabel("Tanimoto similarity (Morgan FP, r=2)")
     ax.set_ylabel("# test spectra")
-    ax.set_title("5 · Tanimoto: oracle vs SIMBA pair", fontweight="bold")
+    ax.set_title("Tanimoto: oracle vs SIMBA pair", fontweight="bold")
     ax.legend(fontsize=9)
     ax.grid(True, alpha=0.2)
 
-    # 6. Oracle rank vs Oracle GT MCES scatter (hexbin)
-    ax = axes2[1]
-    valid = om & (oracle_ranks <= 10000)
-    hb = ax.hexbin(
-        oracle_gt_arr[valid],
-        np.log10(oracle_ranks[valid].astype(float) + 1),
-        gridsize=30,
-        cmap="Blues",
-        mincnt=1,
-    )
-    plt.colorbar(hb, ax=ax, label="count")
-    ax.set_xlabel("Oracle GT MCES")
-    ax.set_ylabel("log10(rank of oracle mol + 1)")
-    ax.set_title("6 · Oracle rank vs GT MCES", fontweight="bold")
-    ax.grid(True, alpha=0.2)
-
-    # 7. SIMBA retrieved GT vs Oracle GT MCES
-    ax = axes2[2]
-    hb2 = ax.hexbin(
-        oracle_gt_arr[om], simba_gt_arr[om], gridsize=30, cmap="Oranges", mincnt=1
-    )
-    plt.colorbar(hb2, ax=ax, label="count")
-    ax.plot([0, 40], [0, 40], "r--", lw=1, label="ideal (SIMBA = oracle)")
-    ax.set_xlabel("Oracle GT MCES")
-    ax.set_ylabel("SIMBA retrieved GT MCES")
-    ax.set_title("7 · SIMBA GT vs Oracle GT MCES", fontweight="bold")
-    ax.legend(fontsize=8)
-    ax.grid(True, alpha=0.2)
-
-    fig2.suptitle("SIMBA Retrieval — Molecular Property Analysis", fontsize=12)
+    fig2.suptitle("SIMBA Retrieval — Tanimoto Analysis", fontsize=12)
     plt.tight_layout()
     plt.savefig(fig2_path, dpi=140, bbox_inches="tight")
     print(f"Figure 2 saved → {fig2_path}")