fix(eval): retry judge timeouts with the request options

LlmAsJudge, HallucinationsV1, and the per-turn quality judge attached HttpRetryOptions and then failed the sample on the first TimeoutError. They now share a tenacity retry that honors those options. Fixes #8075
claxman · Nov 13, 2026 · Nov 13, 2026 · Nov 13, 2026 · b60592ee8f51648867a00330e4db5b0586c15ce9
commit b60592ee8f51648867a00330e4db5b0586c15ce9
diff --git a/src/google/adk/evaluation/_retry_options_utils.py b/src/google/adk/evaluation/_retry_options_utils.py
@@ -14,15 +14,26 @@
 
 from __future__ import annotations
 
+import logging
 from typing import Optional
 
+from google.auth import exceptions as auth_exceptions
 from google.genai import types
+import tenacity
 from typing_extensions import override
 
 from ..agents.callback_context import CallbackContext
+from ..models.base_llm import BaseLlm
 from ..models.llm_request import LlmRequest
 from ..models.llm_response import LlmResponse
 from ..plugins.base_plugin import BasePlugin
+from ..utils.context_utils import Aclosing
+
+logger = logging.getLogger('google_adk.' + __name__)
+
+# google-genai does not retry TimeoutError yet (googleapis/python-genai#3329).
+# When it does, this helper will double-retry and should be removed.
+_JUDGE_TIMEOUT_ERRORS = (TimeoutError, auth_exceptions.TimeoutError)
 
 _RETRY_HTTP_STATUS_CODES = (
     408,  # Request timeout.
@@ -41,6 +52,60 @@
 )
 
 
+def retry_on_judge_timeout(
+    llm_request: LlmRequest,
+) -> tenacity.AsyncRetrying:
+  """Retries a judge call when the client times out.
+
+  Uses the request's ``HttpRetryOptions``. LiteLlm forwards only the attempt
+  count to its own client, so this still covers the other judge models.
+  """
+  options = None
+  config = llm_request.config
+  if config is not None and config.http_options is not None:
+    options = config.http_options.retry_options
+  attempts = 1
+  initial = 0.0
+  max_delay = 0.0
+  exp_base = 1.0
+  jitter = 0.0
+  if options is not None:
+    if options.attempts:
+      attempts = options.attempts
+    if options.initial_delay is not None:
+      initial = options.initial_delay
+    if options.max_delay is not None:
+      max_delay = options.max_delay
+    if options.exp_base is not None:
+      exp_base = options.exp_base
+    if options.jitter is not None:
+      jitter = options.jitter
+  return tenacity.AsyncRetrying(
+      stop=tenacity.stop_after_attempt(attempts),
+      wait=tenacity.wait_exponential_jitter(
+          initial=initial,
+          max=max_delay,
+          exp_base=exp_base,
+          jitter=jitter,
+      ),
+      retry=tenacity.retry_if_exception_type(_JUDGE_TIMEOUT_ERRORS),
+      reraise=True,
+      before_sleep=tenacity.before_sleep_log(logger, logging.INFO),
+  )
+
+
+async def read_judge_response_with_retry(
+    llm: BaseLlm,
+    llm_request: LlmRequest,
+) -> LlmResponse | None:
+  """Returns the first judge response, retrying client timeouts."""
+  async for attempt in retry_on_judge_timeout(llm_request):
+    with attempt:
+      async with Aclosing(llm.generate_content_async(llm_request)) as agen:
+        return await anext(agen, None)
+  return None
+
+
 def add_default_retry_options_if_not_present(llm_request: LlmRequest) -> None:
   """Adds default HTTP Retry Options, if they are not present on the llm_request.
 

diff --git a/src/google/adk/evaluation/hallucinations_v1.py b/src/google/adk/evaluation/hallucinations_v1.py
@@ -31,8 +31,8 @@
 from ..models.llm_request import LlmRequest
 from ..models.registry import LLMRegistry
 from ..utils.content_utils import extract_text_from_content
-from ..utils.context_utils import Aclosing
 from ._retry_options_utils import add_default_retry_options_if_not_present
+from ._retry_options_utils import read_judge_response_with_retry
 from .app_details import AppDetails
 from .eval_case import ConversationScenario
 from .eval_case import Invocation
@@ -540,14 +540,15 @@ async def _evaluate_nl_response(
     )
     add_default_retry_options_if_not_present(segmenter_llm_request)
     try:
-      async with Aclosing(
-          self._judge_model.generate_content_async(segmenter_llm_request)
-      ) as agen:
-        segmenter_response = await agen.__anext__()
-        segmenter_text = get_text_from_content(segmenter_response.content)
-        if segmenter_text is None:
-          return None, "Segmenter returned no text."
-        sentences = _parse_sentences(segmenter_text)
+      segmenter_response = await read_judge_response_with_retry(
+          self._judge_model, segmenter_llm_request
+      )
+      if segmenter_response is None:
+        return None, "Segmenter returned no text."
+      segmenter_text = get_text_from_content(segmenter_response.content)
+      if segmenter_text is None:
+        return None, "Segmenter returned no text."
+      sentences = _parse_sentences(segmenter_text)
     except Exception as e:
       return None, f"Error during sentence segmentation: {e}"
 
@@ -575,14 +576,15 @@ async def _evaluate_nl_response(
     )
     add_default_retry_options_if_not_present(validator_llm_request)
     try:
-      async with Aclosing(
-          self._judge_model.generate_content_async(validator_llm_request)
-      ) as agen:
-        validator_response = await agen.__anext__()
-        validator_text = get_text_from_content(validator_response.content)
-        if validator_text is None:
-          return None, "Sentence validator returned no text."
-        validation_results = _parse_validation_results(validator_text)
+      validator_response = await read_judge_response_with_retry(
+          self._judge_model, validator_llm_request
+      )
+      if validator_response is None:
+        return None, "Sentence validator returned no text."
+      validator_text = get_text_from_content(validator_response.content)
+      if validator_text is None:
+        return None, "Sentence validator returned no text."
+      validation_results = _parse_validation_results(validator_text)
     except Exception as e:
       return None, f"Error during sentence validation: {e}"
 

diff --git a/src/google/adk/evaluation/llm_as_judge.py b/src/google/adk/evaluation/llm_as_judge.py
@@ -24,7 +24,6 @@
 from typing import Optional
 from typing import TypeVar
 
-from google.auth import exceptions as auth_exceptions
 from google.genai import types as genai_types
 from pydantic import ValidationError
 from typing_extensions import override
@@ -33,8 +32,8 @@
 from ..models.llm_request import LlmRequest
 from ..models.llm_response import LlmResponse
 from ..models.registry import LLMRegistry
-from ..utils.context_utils import Aclosing
 from ._retry_options_utils import add_default_retry_options_if_not_present
+from ._retry_options_utils import read_judge_response_with_retry
 from .common import EvalBaseModel
 from .eval_case import ConversationScenario
 from .eval_case import Invocation
@@ -53,26 +52,6 @@
 
 logger = logging.getLogger("google_adk." + __name__)
 
-# Client timeouts are not in google-genai's aiohttp transient set, and a
-# non-Gemini judge never reads http_options.retry_options. Retry them here.
-_JUDGE_TIMEOUT_ERRORS = (TimeoutError, auth_exceptions.TimeoutError)
-
-
-def _judge_timeout_retry(
-    llm_request: LlmRequest,
-) -> tuple[int, float, float, float]:
-  """Returns attempts, delay, exp_base, and max_delay for a judge timeout."""
-  options = None
-  config = llm_request.config
-  if config is not None and config.http_options is not None:
-    options = config.http_options.retry_options
-  if options is None or not options.attempts or options.attempts < 1:
-    return 1, 0.0, 1.0, 0.0
-  initial = options.initial_delay or 0.0
-  exp_base = 1.0 if options.exp_base is None else options.exp_base
-  max_delay = float("inf") if options.max_delay is None else options.max_delay
-  return options.attempts, initial, exp_base, max_delay
-
 
 class AutoRaterScore(EvalBaseModel):
   score: Optional[float] = None
@@ -200,46 +179,24 @@ async def _evaluate_single_sample(
     Returns:
       A PerInvocationResult containing the evaluation score and status.
     """
-    attempts, delay, exp_base, max_delay = _judge_timeout_retry(llm_request)
-    for attempt in range(attempts):
-      try:
-        return await self._read_judge_sample(llm_request, actual, expected)
-      except _JUDGE_TIMEOUT_ERRORS:
-        if attempt + 1 >= attempts:
-          raise
-        await asyncio.sleep(delay)
-        delay = min(delay * exp_base, max_delay)
-    raise RuntimeError(
-        "LLM evaluation failed: no response received from judge model"
+    llm_response = await read_judge_response_with_retry(
+        self._judge_model, llm_request
     )
-
-  async def _read_judge_sample(
-      self,
-      llm_request: LlmRequest,
-      actual: Invocation,
-      expected: Optional[Invocation],
-  ) -> PerInvocationResult:
-    async with Aclosing(
-        self._judge_model.generate_content_async(llm_request)
-    ) as agen:
-      async for llm_response in agen:
-        # Non-streaming call, so there is only one response content.
-        auto_rater_score = (
-            self._convert_auto_rater_response_to_score_for_invocation(
-                llm_response, actual=actual, expected=expected
-            )
-        )
-        return PerInvocationResult(
-            actual_invocation=actual,
-            expected_invocation=expected,
-            score=auto_rater_score.score,
-            eval_status=get_eval_status(
-                auto_rater_score.score, self._threshold
-            ),
-            rubric_scores=auto_rater_score.rubric_scores,
+    if llm_response is None:
+      raise RuntimeError(
+          "LLM evaluation failed: no response received from judge model"
+      )
+    auto_rater_score = (
+        self._convert_auto_rater_response_to_score_for_invocation(
+            llm_response, actual=actual, expected=expected
         )
-    raise RuntimeError(
-        "LLM evaluation failed: no response received from judge model"
+    )
+    return PerInvocationResult(
+        actual_invocation=actual,
+        expected_invocation=expected,
+        score=auto_rater_score.score,
+        eval_status=get_eval_status(auto_rater_score.score, self._threshold),
+        rubric_scores=auto_rater_score.rubric_scores,
     )
 
   @override

diff --git a/src/google/adk/evaluation/simulation/per_turn_user_simulator_quality_v1.py b/src/google/adk/evaluation/simulation/per_turn_user_simulator_quality_v1.py
@@ -27,8 +27,8 @@
 from ...models.llm_request import LlmRequest
 from ...models.llm_response import LlmResponse
 from ...models.registry import LLMRegistry
-from ...utils.context_utils import Aclosing
 from .._retry_options_utils import add_default_retry_options_if_not_present
+from .._retry_options_utils import read_judge_response_with_retry
 from ..eval_case import ConversationScenario
 from ..eval_case import Invocation
 from ..eval_metrics import EvalMetric
@@ -398,8 +398,7 @@ async def _evaluate_stop_signal_turn(
     )
 
   async def _sample_llm(self, llm_request: LlmRequest) -> AutoRaterScore:
-    async with Aclosing(self._llm.generate_content_async(llm_request)) as agen:
-      async for llm_response in agen:
-        # Non-streaming call, so there is only one response content.
-        return self._convert_llm_response_to_score(llm_response)
-    return AutoRaterScore()
+    llm_response = await read_judge_response_with_retry(self._llm, llm_request)
+    if llm_response is None:
+      return AutoRaterScore()
+    return self._convert_llm_response_to_score(llm_response)
diff --git a/tests/unittests/evaluation/test_llm_as_judge.py b/tests/unittests/evaluation/test_llm_as_judge.py
@@ -568,52 +568,6 @@ async def mock_generate_content_async(llm_request):
   assert result.per_invocation_results[0].score == 1.0
 
 
-@pytest.mark.asyncio
-async def test_evaluate_invocations_stops_after_the_timeout_attempts(
-    mock_llm_as_judge, mocker
-):
-  """A judge timeout past the attempt count stays not evaluated."""
-  call_count = 0
-
-  async def mock_generate_content_async(llm_request):
-    nonlocal call_count
-    call_count += 1
-    raise TimeoutError("Request timed out after 120.0 seconds.")
-    yield  # pragma: no cover
-
-  mock_judge_model = mocker.MagicMock()
-  mock_judge_model.generate_content_async = mock_generate_content_async
-  mock_llm_as_judge._judge_model = mock_judge_model
-  mock_llm_as_judge._judge_model_options.num_samples = 1
-  mock_llm_as_judge._judge_model_config = genai_types.GenerateContentConfig(
-      http_options=genai_types.HttpOptions(
-          retry_options=genai_types.HttpRetryOptions(
-              attempts=2, initial_delay=0.0, max_delay=0.0
-          )
-      )
-  )
-  mock_llm_as_judge.aggregate_invocation_results = (
-      lambda per_invocation_results: EvaluationResult(
-          per_invocation_results=per_invocation_results
-      )
-  )
-  invocation = Invocation(
-      invocation_id="id1",
-      user_content=genai_types.Content(parts=[genai_types.Part(text="u1")]),
-      final_response=genai_types.Content(parts=[genai_types.Part(text="r1")]),
-  )
-
-  result = await mock_llm_as_judge.evaluate_invocations(
-      [invocation], [invocation]
-  )
-
-  assert call_count == 2
-  assert (
-      result.per_invocation_results[0].eval_status == EvalStatus.NOT_EVALUATED
-  )
-  assert result.per_invocation_results[0].score is None
-
-
 @pytest.mark.asyncio
 async def test_evaluate_invocations_cancelled_sample(mock_llm_as_judge, mocker):
   """A cancelled sample marks its invocation not evaluated."""