22
33"""Aggregate per-provider JUnit XML test results and generate a trend report.
44
5- Parses ``pytest.xml`` ( JUnit XML) files produced by each CI job, merges them
6- into a single run, combines with historical data, and generates a markdown
7- trend table — the same pattern used by ``scripts/sample_validation/aggregate.py`` .
5+ Parses JUnit XML files produced by CI jobs — both ``pytest.xml`` (Python) and
6+ xunit v3 ``*.junit.xml`` (dotnet) — merges them into a single run, combines
7+ with historical data, and generates a markdown trend table .
88
99Usage (from CI):
1010 python aggregate.py <reports-dir> <history-file> <output-file>
1111
12- The reports directory is expected to contain subdirectories named
13- ``test-results-<provider>/`` each containing a ``pytest.xml`` file
14- (created by ``actions/download-artifact``).
12+ The reports directory is expected to contain artifact subdirectories. Two
13+ layouts are supported:
14+
15+ - **Python (pytest):** ``test-results-<provider>/pytest.xml``
16+ - **Dotnet (xunit):** ``dotnet-test-results-<tfm>-<os>/*.junit.xml``
1517"""
1618
1719from __future__ import annotations
@@ -46,9 +48,21 @@ def _format_run_label(timestamp: str) -> str:
4648def _derive_provider (directory_name : str ) -> str :
4749 """Derive a provider label from a report directory name.
4850
49- ``test-results-openai`` → ``OpenAI``
50- ``test-results-azure-openai`` → ``Azure OpenAI``
51+ Handles both Python and dotnet naming conventions:
52+ - ``test-results-openai`` → ``OpenAI``
53+ - ``test-results-azure-openai`` → ``Azure OpenAI``
54+ - ``dotnet-test-results-net10.0-ubuntu-latest`` → ``net10.0 (ubuntu)``
5155 """
56+ # Dotnet convention: dotnet-test-results-<framework>-<os>
57+ if directory_name .startswith ("dotnet-test-results-" ):
58+ raw = directory_name .replace ("dotnet-test-results-" , "" )
59+ # e.g. "net10.0-ubuntu-latest" → framework="net10.0", os="ubuntu-latest"
60+ parts = raw .split ("-" , 1 )
61+ framework = parts [0 ]
62+ os_label = parts [1 ].split ("-" )[0 ] if len (parts ) > 1 else ""
63+ return f"{ framework } { os_label } if os_label else framework
64+
65+ # Python convention: test-results-<provider>
5266 raw = directory_name .replace ("test-results-" , "" )
5367 known = {
5468 "openai" : "OpenAI" ,
@@ -102,11 +116,21 @@ def _parse_junit_xml(xml_path: Path) -> list[dict[str, str]]:
102116 # it appends the class name, e.g.:
103117 # "packages.foundry.tests.foundry.test_foundry_embedding_client.TestFoundryEmbeddingIntegration"
104118 # We want the file-level module: "test_foundry_embedding_client"
119+ #
120+ # xunit (dotnet) writes classname as the full C# type, e.g.:
121+ # "OpenAIChatCompletion.IntegrationTests.ChatCompletionTests"
122+ # We want the project prefix: "OpenAIChatCompletion"
105123 if classname :
106124 parts = classname .rsplit ("." , 2 )
107125 # If the last segment starts with uppercase it's a class name — take the one before it
108126 if len (parts ) >= 2 and parts [- 1 ][0 :1 ].isupper ():
109- module = parts [- 2 ]
127+ # For dotnet: if the penultimate part is "IntegrationTests" or "UnitTests",
128+ # use the part before that (the project name) instead
129+ if parts [- 2 ] in ("IntegrationTests" , "UnitTests" ) and len (parts ) >= 3 :
130+ # parts[0] may contain dots — take the last segment of it
131+ module = parts [0 ].rsplit ("." , 1 )[- 1 ]
132+ else :
133+ module = parts [- 2 ]
110134 else :
111135 module = parts [- 1 ]
112136 else :
@@ -148,28 +172,61 @@ def _parse_junit_xml(xml_path: Path) -> list[dict[str, str]]:
148172# ---------------------------------------------------------------------------
149173
150174
175+ def _discover_xml_files (reports_dir : Path ) -> list [tuple [str , Path ]]:
176+ """Discover JUnit XML test result files in artifact subdirectories.
177+
178+ Handles two directory layouts:
179+ - **Python (pytest):** ``test-results-<provider>/pytest.xml``
180+ - **Dotnet (xunit):** ``dotnet-test-results-<tfm>-<os>/*.junit.xml``
181+
182+ Returns:
183+ List of ``(directory_name, xml_path)`` tuples.
184+ """
185+ xml_files : list [tuple [str , Path ]] = []
186+ if not reports_dir .is_dir ():
187+ return xml_files
188+
189+ for subdir in sorted (reports_dir .iterdir ()):
190+ if not subdir .is_dir ():
191+ continue
192+
193+ # Python layout: single pytest.xml per artifact
194+ pytest_xml = subdir / "pytest.xml"
195+ if pytest_xml .exists ():
196+ xml_files .append ((subdir .name , pytest_xml ))
197+ continue
198+
199+ # Dotnet layout: multiple *.junit.xml files per artifact
200+ junit_files = sorted (subdir .rglob ("*.junit.xml" ))
201+ for jf in junit_files :
202+ xml_files .append ((subdir .name , jf ))
203+
204+ # Fallback: any .xml file that looks like JUnit (not .trx, not cobertura)
205+ if not junit_files :
206+ for xf in sorted (subdir .rglob ("*.xml" )):
207+ if xf .suffix == ".xml" and not xf .name .endswith (".cobertura.xml" ):
208+ xml_files .append ((subdir .name , xf ))
209+
210+ return xml_files
211+
212+
151213def load_current_run (reports_dir : Path ) -> dict [str , Any ]:
152214 """Load per-provider JUnit XML reports from the current CI run and merge.
153215
216+ Supports both pytest (Python) and xunit v3 (dotnet) JUnit XML formats.
217+
154218 Args:
155- reports_dir: Directory containing ``test-results-<provider>/`` subdirs .
219+ reports_dir: Directory containing artifact subdirectories with XML reports .
156220
157221 Returns:
158222 Merged run dict with ``timestamp``, ``summary``, ``results``.
159223 """
160224 combined_results : dict [str , dict [str , str ]] = {} # nodeid → {status, provider}
161225
162- # actions/download-artifact creates: reports_dir/test-results-openai/pytest.xml
163- xml_files : list [tuple [str , Path ]] = []
164- if reports_dir .is_dir ():
165- for subdir in sorted (reports_dir .iterdir ()):
166- if subdir .is_dir ():
167- xml_file = subdir / "pytest.xml"
168- if xml_file .exists ():
169- xml_files .append ((subdir .name , xml_file ))
226+ xml_files = _discover_xml_files (reports_dir )
170227
171228 if not xml_files :
172- print (f"Warning: No pytest.xml files found in { reports_dir } )
229+ print (f"Warning: No JUnit XML files found in { reports_dir } )
173230 return {
174231 "timestamp" : datetime .now (timezone .utc ).isoformat (),
175232 "summary" : {
@@ -186,7 +243,21 @@ def load_current_run(reports_dir: Path) -> dict[str, Any]:
186243 provider = _derive_provider (dir_name )
187244 tests = _parse_junit_xml (xml_file )
188245 for test in tests :
189- combined_results [test ["nodeid" ]] = {
246+ # Use provider-qualified key when the same test runs under
247+ # multiple providers (e.g. dotnet net10.0 vs net472). This
248+ # prevents later results from silently overwriting earlier ones.
249+ raw_id = test ["nodeid" ]
250+ key = raw_id
251+ if key in combined_results and combined_results [key ]["provider" ] != provider :
252+ # Collision: re-key existing entry and use qualified key for new one
253+ existing = combined_results .pop (key )
254+ combined_results [f"{ existing ['provider' ]} { raw_id } ] = existing
255+ key = f"{ provider } { raw_id }
256+ elif f"{ provider } { raw_id } in combined_results :
257+ # Provider-qualified key already exists (previous collision)
258+ key = f"{ provider } { raw_id }
259+
260+ combined_results [key ] = {
190261 "status" : test ["status" ],
191262 "provider" : provider ,
192263 "module" : test .get ("module" , "" ),
@@ -247,7 +318,7 @@ def _short_name(nodeid: str) -> str:
247318def generate_trend_report (runs : list [dict [str , Any ]]) -> str :
248319 """Generate a markdown trend report from run history."""
249320 lines = [
250- "# 🔬 Flaky Test Report" ,
321+ "# 🔬 Integration Test Report" ,
251322 "" ,
252323 f"*Generated: { datetime .now (timezone .utc ).strftime ('%Y-%m-%d %H:%M UTC' )} ,
253324 "" ,
0 commit comments