feat: show application validation badges

msaroufim · msaroufim · commit f1a7867a4cbe · 2026-07-26T08:43:26.000-07:00
diff --git a/frontend/src/api/api.ts b/frontend/src/api/api.ts
@@ -36,6 +36,13 @@ export interface LeaderboardDetail {
       user_name: string;
       submission_id: number;
       line_count?: number;
+      validation_status?: "completed" | "failed" | null;
+      validation_shapes_passed?: number | null;
+      validation_shapes_total?: number | null;
+      validation_fully_validated?: boolean | null;
+      validation_geomean_speedup?: number | null;
+      validation_contract_version?: string | null;
+      validation_checked_at?: string | null;
     }>
   >;
 }
diff --git a/frontend/src/pages/leaderboard/Leaderboard.test.tsx b/frontend/src/pages/leaderboard/Leaderboard.test.tsx
@@ -108,6 +108,62 @@ describe("Leaderboard", () => {
     expect(screen.getByText(/custom_kernel/)).toBeInTheDocument();
   });
 
+  it("shows full and partial application validation badges beside submitters", () => {
+    const mockData = {
+      deadline: mockDeadline,
+      description: mockDescription,
+      name: mockName,
+      reference: mockReference,
+      starter: mockStarter,
+      gpu_types: ["B200"],
+      rankings: {
+        B200: [
+          {
+            file_name: "fast.py",
+            prev_score: 0,
+            rank: 1,
+            score: 3.25,
+            user_name: "stable-user",
+            submission_id: 101,
+            validation_status: "completed",
+            validation_shapes_passed: 8,
+            validation_shapes_total: 8,
+            validation_fully_validated: true,
+            validation_geomean_speedup: 1.4,
+            validation_contract_version: "v1",
+          },
+          {
+            file_name: "partial.py",
+            prev_score: 0.1,
+            rank: 2,
+            score: 3.5,
+            user_name: "partial-user",
+            submission_id: 102,
+            validation_status: "completed",
+            validation_shapes_passed: 6,
+            validation_shapes_total: 8,
+            validation_fully_validated: false,
+            validation_geomean_speedup: 1.1,
+            validation_contract_version: "v1",
+          },
+        ],
+      },
+    };
+
+    (apiHook.fetcherApiCallback as ReturnType<typeof vi.fn>).mockReturnValue({
+      data: mockData,
+      loading: false,
+      error: null,
+      errorStatus: null,
+      call: mockCall,
+    });
+
+    renderWithRouter(<Leaderboard />);
+
+    expect(screen.getByText("VALIDATED")).toBeInTheDocument();
+    expect(screen.getByText("6/8 VALIDATED")).toBeInTheDocument();
+  });
+
   it("shows loading state", () => {
     (apiHook.fetcherApiCallback as ReturnType<typeof vi.fn>).mockReturnValue({
       data: null,
diff --git a/frontend/src/pages/leaderboard/components/RankingLists.tsx b/frontend/src/pages/leaderboard/components/RankingLists.tsx
@@ -2,8 +2,10 @@ import { useState } from "react";
 import {
   Box,
   Button,
+  Chip,
   Grid,
   Stack,
+  Tooltip,
   type SxProps,
   type Theme,
   Typography,
@@ -30,6 +32,13 @@ interface RankingItem {
   submission_count?: number;
   submission_time?: string;
   line_count?: number;
+  validation_status?: "completed" | "failed" | null;
+  validation_shapes_passed?: number | null;
+  validation_shapes_total?: number | null;
+  validation_fully_validated?: boolean | null;
+  validation_geomean_speedup?: number | null;
+  validation_contract_version?: string | null;
+  validation_checked_at?: string | null;
 }
 
 interface RankingsListProps {
@@ -97,6 +106,58 @@ const styles: Record<string, SxProps<Theme>> = {
   },
 };
 
+function ValidationBadge({ item }: { item: RankingItem }) {
+  if (!item.validation_status) return null;
+
+  const passed = item.validation_shapes_passed ?? 0;
+  const total = item.validation_shapes_total ?? 0;
+  const fullyValidated =
+    item.validation_status === "completed" &&
+    item.validation_fully_validated === true &&
+    total > 0 &&
+    passed === total;
+  const failed = item.validation_status === "failed";
+  const label = fullyValidated
+    ? "VALIDATED"
+    : failed
+      ? "VALIDATION ERROR"
+      : `${passed}/${total} VALIDATED`;
+  const speedup =
+    typeof item.validation_geomean_speedup === "number"
+      ? ` Geomean synchronized-wall speedup across measured shapes: ${item.validation_geomean_speedup.toFixed(2)}×.`
+      : "";
+  const contract = item.validation_contract_version
+    ? ` Contract: ${item.validation_contract_version}.`
+    : "";
+  const checked = item.validation_checked_at
+    ? ` Checked ${new Date(item.validation_checked_at).toLocaleString()}.`
+    : "";
+  const tooltip = fullyValidated
+    ? `All ${passed}/${total} training shapes passed the convergence, numerical, fallback, and speed gates.${speedup}${contract}${checked}`
+    : failed
+      ? `The validation job failed before it could complete.${contract}${checked}`
+      : `${passed}/${total} training shapes passed the convergence, numerical, fallback, and speed gates.${speedup}${contract}${checked}`;
+
+  return (
+    <Tooltip title={tooltip}>
+      <Chip
+        label={label}
+        color={fullyValidated ? "success" : failed ? "error" : "warning"}
+        size="small"
+        variant={fullyValidated ? "filled" : "outlined"}
+        data-testid={`validation-badge-${item.submission_id}`}
+        sx={{
+          height: 20,
+          fontSize: "0.65rem",
+          fontWeight: 800,
+          letterSpacing: "0.03em",
+          flexShrink: 0,
+        }}
+      />
+    </Tooltip>
+  );
+}
+
 export default function RankingsList({
   rankings,
   leaderboardId,
@@ -205,9 +266,19 @@ export default function RankingsList({
                   <Grid size={3}>
                     <Box sx={{ display: "flex", alignItems: "center", gap: 1 }}>
                       <Typography sx={styles.rank}>{item.rank}. </Typography>
-                      <Typography sx={styles.name}>
-                        {item.user_name} {getMedalIcon(item.rank)}
-                      </Typography>
+                      <Box
+                        sx={{
+                          display: "flex",
+                          alignItems: "center",
+                          gap: 0.75,
+                          minWidth: 0,
+                        }}
+                      >
+                        <Typography sx={styles.name}>
+                          {item.user_name} {getMedalIcon(item.rank)}
+                        </Typography>
+                        <ValidationBadge item={item} />
+                      </Box>
                     </Box>
                   </Grid>
                   <Grid size={scoreSize}>
diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py
@@ -265,11 +265,38 @@ def _get_query():
                 s.file_name AS file_name,
                 r.submission_id AS submission_id,
                 COALESCE(sc.submission_count, 0) AS submission_count,
+                validation.status AS validation_status,
+                validation.passed_shapes AS validation_shapes_passed,
+                validation.total_shapes AS validation_shapes_total,
+                validation.fully_validated AS validation_fully_validated,
+                validation.geomean_sync_wall_speedup AS validation_geomean_speedup,
+                validation.contract_version AS validation_contract_version,
+                validation.checked_at AS validation_checked_at,
                 RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank
             FROM leaderboard.runs r
                 JOIN leaderboard.submission s ON r.submission_id = s.id
                 LEFT JOIN leaderboard.user_info u ON s.user_id = u.id
                 LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner
+                LEFT JOIN LATERAL (
+                    SELECT
+                        status,
+                        passed_shapes,
+                        total_shapes,
+                        fully_validated,
+                        geomean_sync_wall_speedup,
+                        contract_version,
+                        checked_at
+                    FROM leaderboard.submission_validation
+                    WHERE submission_id = s.id
+                        AND gpu_type = r.runner
+                        AND contract_version = (
+                            SELECT task->'validation'->>'version'
+                            FROM leaderboard.leaderboard
+                            WHERE id = %(leaderboard_id)s
+                        )
+                    ORDER BY checked_at DESC
+                    LIMIT 1
+                ) validation ON TRUE
             WHERE NOT r.secret
                 AND r.score IS NOT NULL
                 AND r.passed
@@ -312,7 +339,14 @@ def _get_query():
                         'file_name', r.file_name,
                         'submission_id', r.submission_id,
                         'submission_count', r.submission_count,
-                        'submission_time', r.submission_time
+                        'submission_time', r.submission_time,
+                        'validation_status', r.validation_status,
+                        'validation_shapes_passed', r.validation_shapes_passed,
+                        'validation_shapes_total', r.validation_shapes_total,
+                        'validation_fully_validated', r.validation_fully_validated,
+                        'validation_geomean_speedup', r.validation_geomean_speedup,
+                        'validation_contract_version', r.validation_contract_version,
+                        'validation_checked_at', r.validation_checked_at
                     )
                     ORDER BY r.score ASC
                 )
diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py
@@ -49,6 +49,54 @@ def test_active_leaderboard_omits_line_counts(client, app):
     assert all("line_count" not in item for item in ranked_items)
 
 
+def test_leaderboard_includes_latest_validation_summary(client, app):
+    initial = client.get("/api/leaderboard/339").get_json()
+    gpu_type, rankings = next(iter(initial["data"]["rankings"].items()))
+    submission_id = rankings[0]["submission_id"]
+
+    with app.app_context():
+        conn = get_db_connection()
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                UPDATE leaderboard.leaderboard
+                SET task = jsonb_set(
+                    task,
+                    '{validation}',
+                    '{"version": "v1"}'::jsonb
+                )
+                WHERE id = 339
+                """
+            )
+            cur.execute(
+                """
+                INSERT INTO leaderboard.submission_validation (
+                    submission_id, gpu_type, contract_name, contract_version,
+                    status, passed_shapes, total_shapes, fully_validated,
+                    geomean_sync_wall_speedup, result
+                )
+                VALUES (%s, %s, 'natural-gradient-training', 'v1',
+                        'completed', 8, 8, TRUE, 1.25, '{}')
+                """,
+                (submission_id, gpu_type),
+            )
+            conn.commit()
+
+    response = client.get("/api/leaderboard/339")
+    assert response.status_code == 200
+    validated = next(
+        item
+        for item in response.get_json()["data"]["rankings"][gpu_type]
+        if item["submission_id"] == submission_id
+    )
+    assert validated["validation_status"] == "completed"
+    assert validated["validation_shapes_passed"] == 8
+    assert validated["validation_shapes_total"] == 8
+    assert validated["validation_fully_validated"] is True
+    assert validated["validation_geomean_speedup"] == 1.25
+    assert validated["validation_contract_version"] == "v1"
+
+
 def test_leaderboard_line_counts_support_bytea_code_storage(client, app):
     with app.app_context():
         conn = get_db_connection()
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,3 +1,4 @@
+import os
 import random
 import secrets
 import string
@@ -131,7 +132,7 @@ def db_server():
                 "-f",
                 "tests/data.sql",
             ],
-            env={"PGPASSWORD": password},
+            env={**os.environ, "PGPASSWORD": password},
         )
 
         if result.returncode != 0:
diff --git a/tests/data.sql b/tests/data.sql
@@ -9219,6 +9219,24 @@ CREATE TABLE IF NOT EXISTS leaderboard.submission_job_status (
         CONSTRAINT uq_submission_job_status_submission_id
             UNIQUE (submission_id)                            -- one-to-one with submission
     );
+
+CREATE TABLE IF NOT EXISTS leaderboard.submission_validation (
+    id BIGSERIAL PRIMARY KEY,
+    submission_id INTEGER NOT NULL
+        REFERENCES leaderboard.submission(id) ON DELETE CASCADE,
+    gpu_type TEXT NOT NULL,
+    contract_name TEXT NOT NULL,
+    contract_version TEXT NOT NULL,
+    status TEXT NOT NULL CHECK (status IN ('completed', 'failed')),
+    passed_shapes INTEGER NOT NULL DEFAULT 0,
+    total_shapes INTEGER NOT NULL DEFAULT 0,
+    fully_validated BOOLEAN NOT NULL DEFAULT FALSE,
+    geomean_sync_wall_speedup DOUBLE PRECISION,
+    result JSONB NOT NULL DEFAULT '{}'::jsonb,
+    error TEXT,
+    checked_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    UNIQUE (submission_id, gpu_type, contract_version)
+);
 --
 -- PostgreSQL database dump complete
 --
