Skip to content

Commit 550a771

Browse files
fix(evaluation): skip invocations without user events in convert_events_to_eval_invocations
Sessions can contain invocation_ids whose events are all authored by agents or tools (e.g. internal/background turns with no corresponding user message). Previously, convert_events_to_eval_invocations left user_content as an empty Content(parts=[]) for such invocations, and earlier versions used an empty string, which caused a Pydantic ValidationError because Invocation.user_content requires a genai_types.Content object. Invocations without a user-authored event are not meaningful for evaluation, so skip them instead of constructing an Invocation with a placeholder user_content. A debug log line is emitted for each skipped invocation to aid troubleshooting. Fixes #3760 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent abcf14c commit 550a771

File tree

2 files changed

+59
-1
lines changed

2 files changed

+59
-1
lines changed

src/google/adk/evaluation/evaluation_generator.py

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ def convert_events_to_eval_invocations(
281281
for invocation_id, events in events_by_invocation_id.items():
282282
final_response = None
283283
final_event = None
284-
user_content = Content(parts=[])
284+
user_content = None
285285
invocation_timestamp = 0
286286
app_details = None
287287
if (
@@ -312,6 +312,18 @@ def convert_events_to_eval_invocations(
312312
events_to_add.append(event)
313313
break
314314

315+
if user_content is None:
316+
# Skip invocations that have no user-authored event. Such invocations
317+
# arise from internal/system-driven turns (e.g. background agent tasks)
318+
# and are not meaningful for evaluation purposes. Including them would
319+
# also cause a Pydantic ValidationError because Invocation.user_content
320+
# requires a Content object.
321+
logger.debug(
322+
'Skipping invocation %s: no user-authored event found.',
323+
invocation_id,
324+
)
325+
continue
326+
315327
invocation_events = [
316328
InvocationEvent(author=e.author, content=e.content)
317329
for e in events_to_add

tests/unittests/evaluation/test_evaluation_generator.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,52 @@ def test_convert_multi_agent_final_responses(
226226
assert intermediate_events[0].author == "agent1"
227227
assert intermediate_events[0].content.parts[0].text == "First response"
228228

229+
def test_invocation_without_user_event_is_skipped(self):
230+
"""Invocations with no user-authored event must be skipped.
231+
232+
Regression test for https://github.com/google/adk-python/issues/3760.
233+
When a session contains an invocation_id whose events are all authored by
234+
agents or tools (no 'user' event), convert_events_to_eval_invocations used
235+
to leave user_content as a bare string, causing a Pydantic ValidationError
236+
from Invocation.user_content which requires genai_types.Content.
237+
The fix skips such invocations because they represent internal/system-driven
238+
turns that are not meaningful for evaluation.
239+
"""
240+
events = [
241+
_build_event("agent", [types.Part(text="agent-only event")], "inv1"),
242+
]
243+
244+
# Must not raise a Pydantic ValidationError.
245+
invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
246+
247+
assert invocations == [], (
248+
"Invocations without a user event should be skipped."
249+
)
250+
251+
def test_mixed_invocations_skips_only_agent_only_ones(self):
252+
"""Only agent-only invocations are skipped; normal invocations are kept.
253+
254+
Regression test for https://github.com/google/adk-python/issues/3760.
255+
"""
256+
events = [
257+
# inv1: normal user+agent turn — should be kept.
258+
_build_event("user", [types.Part(text="Hello")], "inv1"),
259+
_build_event("agent", [types.Part(text="Hi there!")], "inv1"),
260+
# inv2: agent-only turn (e.g. background/system task) — should be skipped.
261+
_build_event("agent", [types.Part(text="Internal work")], "inv2"),
262+
# inv3: normal user+agent turn — should be kept.
263+
_build_event("user", [types.Part(text="Follow-up")], "inv3"),
264+
_build_event("agent", [types.Part(text="Sure!")], "inv3"),
265+
]
266+
267+
invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
268+
269+
assert len(invocations) == 2
270+
assert invocations[0].invocation_id == "inv1"
271+
assert invocations[0].user_content.parts[0].text == "Hello"
272+
assert invocations[1].invocation_id == "inv3"
273+
assert invocations[1].user_content.parts[0].text == "Follow-up"
274+
229275

230276
class TestGetAppDetailsByInvocationId:
231277
"""Test cases for EvaluationGenerator._get_app_details_by_invocation_id method."""

0 commit comments

Comments
 (0)