Skip to content

Commit bd048b5

Browse files
msaroufimmsaroufim
authored
Show per-submission application validation badges (#261)
* feat: show application validation badges * fix: repair leaderboard validation coverage * fix: describe validation gates accurately * test: keep unvalidated entries ranked
1 parent 0c5fbc4 commit bd048b5

8 files changed

Lines changed: 296 additions & 19 deletions

File tree

‎frontend/src/api/api.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@ export interface LeaderboardDetail {
3636
user_name: string;
3737
submission_id: number;
3838
line_count?: number;
39+
validation_status?: "completed" | "failed" | null;
40+
validation_shapes_passed?: number | null;
41+
validation_shapes_total?: number | null;
42+
validation_fully_validated?: boolean | null;
43+
validation_geomean_speedup?: number | null;
44+
validation_contract_version?: string | null;
45+
validation_checked_at?: string | null;
3946
}>
4047
>;
4148
}

‎frontend/src/pages/leaderboard/Leaderboard.test.tsx‎

Lines changed: 71 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ vi.mock("../../lib/hooks/useApi", () => ({
99
fetcherApiCallback: vi.fn(),
1010
}));
1111

12+
vi.mock("./components/UserTrendChart", () => ({
13+
default: () => <div data-testid="user-trend-chart" />,
14+
}));
15+
1216
// Mutable auth state for mocking useAuthStore per test
1317
type AuthState = {
1418
me: null | { authenticated: boolean; user?: { identity?: string } };
@@ -108,6 +112,62 @@ describe("Leaderboard", () => {
108112
expect(screen.getByText(/custom_kernel/)).toBeInTheDocument();
109113
});
110114

115+
it("shows full and partial application validation badges beside submitters", () => {
116+
const mockData = {
117+
deadline: mockDeadline,
118+
description: mockDescription,
119+
name: mockName,
120+
reference: mockReference,
121+
starter: mockStarter,
122+
gpu_types: ["B200"],
123+
rankings: {
124+
B200: [
125+
{
126+
file_name: "fast.py",
127+
prev_score: 0,
128+
rank: 1,
129+
score: 3.25,
130+
user_name: "stable-user",
131+
submission_id: 101,
132+
validation_status: "completed",
133+
validation_shapes_passed: 8,
134+
validation_shapes_total: 8,
135+
validation_fully_validated: true,
136+
validation_geomean_speedup: 1.4,
137+
validation_contract_version: "v1",
138+
},
139+
{
140+
file_name: "partial.py",
141+
prev_score: 0.1,
142+
rank: 2,
143+
score: 3.5,
144+
user_name: "partial-user",
145+
submission_id: 102,
146+
validation_status: "completed",
147+
validation_shapes_passed: 6,
148+
validation_shapes_total: 8,
149+
validation_fully_validated: false,
150+
validation_geomean_speedup: 1.1,
151+
validation_contract_version: "v1",
152+
},
153+
],
154+
},
155+
};
156+
157+
(apiHook.fetcherApiCallback as ReturnType<typeof vi.fn>).mockReturnValue({
158+
data: mockData,
159+
loading: false,
160+
error: null,
161+
errorStatus: null,
162+
call: mockCall,
163+
});
164+
165+
renderWithRouter(<Leaderboard />);
166+
167+
expect(screen.getByText("VALIDATED")).toBeInTheDocument();
168+
expect(screen.getByText("6/8 VALIDATED")).toBeInTheDocument();
169+
});
170+
111171
it("shows loading state", () => {
112172
(apiHook.fetcherApiCallback as ReturnType<typeof vi.fn>).mockReturnValue({
113173
data: null,
@@ -118,7 +178,7 @@ describe("Leaderboard", () => {
118178
});
119179

120180
renderWithRouter(<Leaderboard />);
121-
expect(screen.getByText(/Summoning/i)).toBeInTheDocument();
181+
expect(screen.getByRole("progressbar")).toBeInTheDocument();
122182
});
123183

124184
it("shows error message", () => {
@@ -264,17 +324,18 @@ describe("Leaderboard", () => {
264324
const btn = screen.queryByTestId("ranking-show-all-button-0");
265325
expect(btn).toBeInTheDocument();
266326

267-
// By default only 3 rows shown
268-
expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(3);
269-
270-
// Click to show all
271-
fireEvent.click(btn!);
327+
// Rankings start expanded.
272328
expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(4);
273329
expect(within(btn!).getByText(/Hide/i)).toBeInTheDocument();
274330

275-
// Click to hide again
331+
// Hide the rows after the top three.
276332
fireEvent.click(btn!);
277333
expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(3);
334+
expect(within(btn!).getByText(/Show all/i)).toBeInTheDocument();
335+
336+
// Expand again.
337+
fireEvent.click(btn!);
338+
expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(4);
278339
});
279340

280341
// -------------------- Starter codeblock --------------------
@@ -510,9 +571,9 @@ describe("Leaderboard", () => {
510571
// Switch to the Submission tab explicitly
511572
fireEvent.click(screen.getByRole("tab", { name: /Submission/i }));
512573

513-
const submit_btn = screen.getByTestId("leaderboard-submit-btn");
514-
expect(submit_btn).toBeInTheDocument();
515-
expect(submit_btn).toBeDisabled();
574+
expect(
575+
screen.queryByTestId("leaderboard-submit-btn"),
576+
).not.toBeInTheDocument();
516577

517578
const deadline_txt = screen.getByTestId("deadline-passed-text");
518579
expect(

‎frontend/src/pages/leaderboard/Leaderboard.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,8 @@ const LeaderboardContent = memo(function LeaderboardContent() {
178178
findTopUsers();
179179
}, [id, data?.rankings]);
180180

181-
if (loading || !data) return <Loading />;
182181
if (error) return <ErrorAlert status={errorStatus} message={error} />;
182+
if (loading || !data) return <Loading />;
183183
if (!data) return null;
184184

185185
const toDeadlineLocal = (raw: string) => {

‎frontend/src/pages/leaderboard/components/RankingLists.tsx‎

Lines changed: 77 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ import { useState } from "react";
22
import {
33
Box,
44
Button,
5+
Chip,
56
Grid,
67
Stack,
8+
Tooltip,
79
type SxProps,
810
type Theme,
911
Typography,
@@ -30,6 +32,13 @@ interface RankingItem {
3032
submission_count?: number;
3133
submission_time?: string;
3234
line_count?: number;
35+
validation_status?: "completed" | "failed" | null;
36+
validation_shapes_passed?: number | null;
37+
validation_shapes_total?: number | null;
38+
validation_fully_validated?: boolean | null;
39+
validation_geomean_speedup?: number | null;
40+
validation_contract_version?: string | null;
41+
validation_checked_at?: string | null;
3342
}
3443

3544
interface RankingsListProps {
@@ -97,6 +106,58 @@ const styles: Record<string, SxProps<Theme>> = {
97106
},
98107
};
99108

109+
function ValidationBadge({ item }: { item: RankingItem }) {
110+
if (!item.validation_status) return null;
111+
112+
const passed = item.validation_shapes_passed ?? 0;
113+
const total = item.validation_shapes_total ?? 0;
114+
const fullyValidated =
115+
item.validation_status === "completed" &&
116+
item.validation_fully_validated === true &&
117+
total > 0 &&
118+
passed === total;
119+
const failed = item.validation_status === "failed";
120+
const label = fullyValidated
121+
? "VALIDATED"
122+
: failed
123+
? "VALIDATION ERROR"
124+
: `${passed}/${total} VALIDATED`;
125+
const speedup =
126+
typeof item.validation_geomean_speedup === "number"
127+
? ` Geomean synchronized-wall speedup across measured shapes: ${item.validation_geomean_speedup.toFixed(2)}×.`
128+
: "";
129+
const contract = item.validation_contract_version
130+
? ` Contract: ${item.validation_contract_version}.`
131+
: "";
132+
const checked = item.validation_checked_at
133+
? ` Checked ${new Date(item.validation_checked_at).toLocaleString()}.`
134+
: "";
135+
const tooltip = fullyValidated
136+
? `All ${passed}/${total} training shapes passed the convergence, numerical, and speed gates.${speedup}${contract}${checked}`
137+
: failed
138+
? `The validation job failed before it could complete.${contract}${checked}`
139+
: `${passed}/${total} training shapes passed the convergence, numerical, and speed gates.${speedup}${contract}${checked}`;
140+
141+
return (
142+
<Tooltip title={tooltip}>
143+
<Chip
144+
label={label}
145+
color={fullyValidated ? "success" : failed ? "error" : "warning"}
146+
size="small"
147+
variant={fullyValidated ? "filled" : "outlined"}
148+
data-testid={`validation-badge-${item.submission_id}`}
149+
sx={{
150+
height: 20,
151+
fontSize: "0.65rem",
152+
fontWeight: 800,
153+
letterSpacing: "0.03em",
154+
flexShrink: 0,
155+
}}
156+
/>
157+
</Tooltip>
158+
);
159+
}
160+
100161
export default function RankingsList({
101162
rankings,
102163
leaderboardId,
@@ -114,7 +175,7 @@ export default function RankingsList({
114175
const toggleExpanded = (field: string) => {
115176
setExpanded((prev) => ({
116177
...prev,
117-
[field]: !prev[field],
178+
[field]: !(prev[field] ?? true),
118179
}));
119180
};
120181

@@ -205,9 +266,19 @@ export default function RankingsList({
205266
<Grid size={3}>
206267
<Box sx={{ display: "flex", alignItems: "center", gap: 1 }}>
207268
<Typography sx={styles.rank}>{item.rank}. </Typography>
208-
<Typography sx={styles.name}>
209-
{item.user_name} {getMedalIcon(item.rank)}
210-
</Typography>
269+
<Box
270+
sx={{
271+
display: "flex",
272+
alignItems: "center",
273+
gap: 0.75,
274+
minWidth: 0,
275+
}}
276+
>
277+
<Typography sx={styles.name}>
278+
{item.user_name} {getMedalIcon(item.rank)}
279+
</Typography>
280+
<ValidationBadge item={item} />
281+
</Box>
211282
</Box>
212283
</Grid>
213284
<Grid size={scoreSize}>
@@ -222,7 +293,7 @@ export default function RankingsList({
222293
</Typography>
223294
</Grid>
224295
<Grid size={isAdmin ? 2 : fileSize}>
225-
<Typography
296+
<Box
226297
sx={{
227298
overflow: "hidden",
228299
textOverflow: "ellipsis",
@@ -253,7 +324,7 @@ export default function RankingsList({
253324
{item.file_name}
254325
</Button>
255326
)}
256-
</Typography>
327+
</Box>
257328
</Grid>
258329
{showLineCount && (
259330
<Grid size={isAdmin ? 1 : 2}>

‎kernelboard/api/leaderboard.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,11 +265,38 @@ def _get_query():
265265
s.file_name AS file_name,
266266
r.submission_id AS submission_id,
267267
COALESCE(sc.submission_count, 0) AS submission_count,
268+
validation.status AS validation_status,
269+
validation.passed_shapes AS validation_shapes_passed,
270+
validation.total_shapes AS validation_shapes_total,
271+
validation.fully_validated AS validation_fully_validated,
272+
validation.geomean_sync_wall_speedup AS validation_geomean_speedup,
273+
validation.contract_version AS validation_contract_version,
274+
validation.checked_at AS validation_checked_at,
268275
RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank
269276
FROM leaderboard.runs r
270277
JOIN leaderboard.submission s ON r.submission_id = s.id
271278
LEFT JOIN leaderboard.user_info u ON s.user_id = u.id
272279
LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner
280+
LEFT JOIN LATERAL (
281+
SELECT
282+
status,
283+
passed_shapes,
284+
total_shapes,
285+
fully_validated,
286+
geomean_sync_wall_speedup,
287+
contract_version,
288+
checked_at
289+
FROM leaderboard.submission_validation
290+
WHERE submission_id = s.id
291+
AND gpu_type = r.runner
292+
AND contract_version = (
293+
SELECT task->'validation'->>'version'
294+
FROM leaderboard.leaderboard
295+
WHERE id = %(leaderboard_id)s
296+
)
297+
ORDER BY checked_at DESC
298+
LIMIT 1
299+
) validation ON TRUE
273300
WHERE NOT r.secret
274301
AND r.score IS NOT NULL
275302
AND r.passed
@@ -312,7 +339,14 @@ def _get_query():
312339
'file_name', r.file_name,
313340
'submission_id', r.submission_id,
314341
'submission_count', r.submission_count,
315-
'submission_time', r.submission_time
342+
'submission_time', r.submission_time,
343+
'validation_status', r.validation_status,
344+
'validation_shapes_passed', r.validation_shapes_passed,
345+
'validation_shapes_total', r.validation_shapes_total,
346+
'validation_fully_validated', r.validation_fully_validated,
347+
'validation_geomean_speedup', r.validation_geomean_speedup,
348+
'validation_contract_version', r.validation_contract_version,
349+
'validation_checked_at', r.validation_checked_at
316350
)
317351
ORDER BY r.score ASC
318352
)

0 commit comments

Comments
 (0)