From 3db17c2c7cf8685467ec3dac20366c9c7a25195f Mon Sep 17 00:00:00 2001
From: Ivan Despot <66276597+g-despot@users.noreply.github.com>
Date: Mon, 6 Jul 2026 14:24:07 +0200
Subject: [PATCH 1/4] test(indexability): follow llms.txt redirect to
 weaviate.io; skip on network reset

/llms.txt 301-redirects to weaviate.io/llms.txt. Add weaviate.io to the Claude
web_fetch allowed_domains so it can follow the redirect, and make
test_llms_txt_accessible use a browser UA and treat a transient connection
reset/timeout as an inconclusive skip (matching test_llms_txt_code.py) rather
than a hard failure.
---
 tests/test_docs_indexability.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/tests/test_docs_indexability.py b/tests/test_docs_indexability.py
index 51c612fd..fc6f00ba 100644
--- a/tests/test_docs_indexability.py
+++ b/tests/test_docs_indexability.py
@@ -277,11 +277,21 @@ def test_llm_notice_present(path):
 @pytest.mark.indexability
 def test_llms_txt_accessible():
     """/llms.txt returns 200, has substantial content, and mentions Weaviate."""
-    resp = requests.get(
-        f"{BASE_URL}/llms.txt",
-        timeout=30,
-        headers={"User-Agent": "WeaviateDocsIndexabilityTest/1.0"},
-    )
+    try:
+        resp = requests.get(
+            f"{BASE_URL}/llms.txt",
+            timeout=30,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/124.0.0.0 Safari/537.36"
+                )
+            },
+        )
+    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc:
+        # network failure must not flake the suite
+        pytest.skip(f"Network issue fetching /llms.txt: {exc}")
     assert resp.status_code == 200, f"/llms.txt returned {resp.status_code}"
     assert len(resp.text) > 500, f"/llms.txt content too short ({len(resp.text)} chars)"
     assert "weaviate" in resp.text.lower(), "/llms.txt doesn't mention Weaviate"
@@ -450,7 +460,7 @@ def test_claude_can_fetch_llms_txt():
             "type": "web_fetch_20250910",
             "name": "web_fetch",
             "max_uses": 1,
-            "allowed_domains": ["docs.weaviate.io"],
+            "allowed_domains": ["docs.weaviate.io", "weaviate.io"],
         }],
         messages=[{
             "role": "user",

From a8f68e577dc23b0a522518ceed41502527090726 Mon Sep 17 00:00:00 2001
From: Ivan Despot <66276597+g-despot@users.noreply.github.com>
Date: Tue, 7 Jul 2026 10:00:36 +0200
Subject: [PATCH 2/4] Revert "docs(query-agent): make search-mode filtering a
 required argument"

This reverts commit 7d55f67386ecb32f3150a9eddda78d72b848790e.
---
 _includes/code/llms-txt/python/query_agent.py             | 2 +-
 _includes/code/python/quickstart.short.query_agent.py     | 2 +-
 docs/query-agent/_includes/code/query_agent.mts           | 4 ----
 docs/query-agent/_includes/code/query_agent.py            | 6 ++----
 .../query-agent/_includes/code/query_agent_get_started.py | 1 -
 docs/query-agent/_includes/code/quickstart.mts            | 1 -
 docs/query-agent/_includes/code/quickstart.py             | 3 +--
 docs/query-agent/_includes/code/search_mode.mts           | 3 ---
 docs/query-agent/_includes/code/search_mode.py            | 5 -----
 docs/query-agent/guides/search_mode.md                    | 8 +++-----
 10 files changed, 8 insertions(+), 27 deletions(-)

diff --git a/_includes/code/llms-txt/python/query_agent.py b/_includes/code/llms-txt/python/query_agent.py
index 77a26afb..7b2c3412 100644
--- a/_includes/code/llms-txt/python/query_agent.py
+++ b/_includes/code/llms-txt/python/query_agent.py
@@ -28,7 +28,7 @@
 print(response.final_answer)
 
 # Retrieval only (no generation)
-search_response = qa.search("sci-fi movies", filtering="recall", limit=5)
+search_response = qa.search("sci-fi movies", limit=5)
 # END llms_query_agent
 
 assert response.final_answer
diff --git a/_includes/code/python/quickstart.short.query_agent.py b/_includes/code/python/quickstart.short.query_agent.py
index 05663434..07cafb32 100644
--- a/_includes/code/python/quickstart.short.query_agent.py
+++ b/_includes/code/python/quickstart.short.query_agent.py
@@ -18,7 +18,7 @@
     qa = QueryAgent(client=client, collections=["Movie"])
 
     # Step 2.3: Perform a query using Search Mode
-    response = qa.search("Find a cool sci-fi movie.", filtering="recall", limit=1)
+    response = qa.search("Find a cool sci-fi movie.", limit=1)
     # highlight-end
 
     # Print the response
diff --git a/docs/query-agent/_includes/code/query_agent.mts b/docs/query-agent/_includes/code/query_agent.mts
index f5a042d5..61cd6890 100644
--- a/docs/query-agent/_includes/code/query_agent.mts
+++ b/docs/query-agent/_includes/code/query_agent.mts
@@ -141,7 +141,6 @@ clothingResponse.display();
 // START BasicSearchQuery
 // Perform a search using Search Mode (retrieval only, no answer generation)
 const basicSearchResponse = await qa.search("Find me some vintage shoes under $70", {
-    filtering: "recall",
     limit: 10
 })
 
@@ -155,7 +154,6 @@ for (const obj of basicSearchResponse.searchResults.objects) {
 // Diversity ranking needs a vectorizer it can resolve. Scope the call to a
 // single collection with a target vector so the agent knows what to use.
 const diversitySearchResponse = await qa.search("summer shoes", {
-    filtering: "recall",
     limit: 10,
     diversityWeight: 0.5,
     collections: [{
@@ -181,7 +179,6 @@ basicResponse.display();
 // START SearchModeResponseStructure
 // SearchModeResponse structure for TypeScript
 const searchResponse = await qa.search("winter boots for under $100", {
-    filtering: "recall",
     limit: 5
 })
 
@@ -209,7 +206,6 @@ for (const obj of searchResponse.searchResults.objects) {
 // Search with pagination
 const responsePage1 = await qa.search(
     "Find summer shoes and accessories between $50 and $100 that have the tag 'sale'", {
-    filtering: "recall",
     limit: 3,
 })
 
diff --git a/docs/query-agent/_includes/code/query_agent.py b/docs/query-agent/_includes/code/query_agent.py
index bd58dac5..e8df3d9c 100644
--- a/docs/query-agent/_includes/code/query_agent.py
+++ b/docs/query-agent/_includes/code/query_agent.py
@@ -211,7 +211,7 @@
 
 # START BasicSearchQuery
 # Perform a search using Search Mode (retrieval only, no answer generation)
-search_response = qa.search("Find me some vintage shoes under $70", filtering="recall", limit=10)
+search_response = qa.search("Find me some vintage shoes under $70", limit=10)
 
 # Access the search results
 for obj in search_response.search_results.objects:
@@ -220,7 +220,7 @@
 
 # START SearchModeResponseStructure
 # SearchModeResponse structure for Python
-search_response = qa.search("winter boots for under $100", filtering="recall", limit=5)
+search_response = qa.search("winter boots for under $100", limit=5)
 
 # Access different parts of the response
 print(f"Original query: {search_response.searches[0].query}")
@@ -244,7 +244,6 @@
 # Search with pagination
 response_page_1 = qa.search(
     "Find summer shoes and accessories between $50 and $100 that have the tag 'sale'",
-    filtering="recall",
     limit=3,
 )
 
@@ -270,7 +269,6 @@
 # START DiversityRanking
 search_response = qa.search(
     "summer shoes",
-    filtering="recall",
     limit=10,
     diversity_weight=0.5,
     collections=[
diff --git a/docs/query-agent/_includes/code/query_agent_get_started.py b/docs/query-agent/_includes/code/query_agent_get_started.py
index 9abaea4b..2cf55a8d 100644
--- a/docs/query-agent/_includes/code/query_agent_get_started.py
+++ b/docs/query-agent/_includes/code/query_agent_get_started.py
@@ -135,7 +135,6 @@
 # START SearchMode
 search_response = agent.search(
     "Find me some vintage shoes under $70",
-    filtering="recall",
     limit=10,
 )
 
diff --git a/docs/query-agent/_includes/code/quickstart.mts b/docs/query-agent/_includes/code/quickstart.mts
index 0693851c..727b5c93 100644
--- a/docs/query-agent/_includes/code/quickstart.mts
+++ b/docs/query-agent/_includes/code/quickstart.mts
@@ -51,7 +51,6 @@ qa = new QueryAgent(client, {
 // START BasicSearchQuery
 const searchResponse = await qa.search(
     "Find me some vintage shoes under $70", {
-    filtering: "recall",
     limit: 10,
 });
 // END BasicSearchQuery
diff --git a/docs/query-agent/_includes/code/quickstart.py b/docs/query-agent/_includes/code/quickstart.py
index 013cc3ce..73af640c 100644
--- a/docs/query-agent/_includes/code/quickstart.py
+++ b/docs/query-agent/_includes/code/quickstart.py
@@ -53,8 +53,7 @@
 
 # START BasicSearchQuery
 search_response = qa.search(
-    "Find me some vintage shoes under $70",
-    filtering="recall",
+    "Find me some vintage shoes under $70", 
     limit=10
 )
 # END BasicSearchQuery
diff --git a/docs/query-agent/_includes/code/search_mode.mts b/docs/query-agent/_includes/code/search_mode.mts
index e07231ec..fc9f355b 100644
--- a/docs/query-agent/_includes/code/search_mode.mts
+++ b/docs/query-agent/_includes/code/search_mode.mts
@@ -39,7 +39,6 @@ await populateWeaviate(client);
 
 // START BasicSearchMode
 const searchResponse = await qa.search("Find me some vintage shoes under $70", {
-    filtering: "recall",
     limit: 10,
 });
 
@@ -51,7 +50,6 @@ for (const obj of searchResponse.searchResults.objects) {
 
 // START DiversityRanking
 const diversitySearchResponse = await qa.search("summer shoes", {
-    filtering: "recall",
     limit: 10,
     diversityWeight: 0.5,
     collections: [{
@@ -69,7 +67,6 @@ for (const obj of diversitySearchResponse.searchResults.objects) {
 // Search with pagination
 const responsePage1 = await qa.search(
     "Find summer shoes and accessories between $50 and $100 that have the tag 'sale'", {
-    filtering: "recall",
     limit: 3,
 });
 
diff --git a/docs/query-agent/_includes/code/search_mode.py b/docs/query-agent/_includes/code/search_mode.py
index 281fdb03..d1bab68d 100644
--- a/docs/query-agent/_includes/code/search_mode.py
+++ b/docs/query-agent/_includes/code/search_mode.py
@@ -41,7 +41,6 @@
 # START BasicSearchMode
 search_response = qa.search(
     query="Find me some vintage shoes under $70",
-    filtering="recall",
     limit=10,
 )
 
@@ -63,7 +62,6 @@
 
 search_response = qa.search(
     "summer shoes",
-    filtering="recall",
     limit=10,
     diversity_weight=0.5,
 )
@@ -76,7 +74,6 @@
 # Search with pagination
 response_page_1 = qa.search(
     "Find summer shoes and accessories between $50 and $100 that have the tag 'sale'",
-    filtering="recall",
     limit=3,
 )
 
@@ -138,7 +135,6 @@
 # START AsyncSearch
 await async_qa.search(
     query="Find me some vintage shoes under $70",
-    filtering="recall",
     limit=10,
 )
 # END AsyncSearch
@@ -165,7 +161,6 @@ async def _async_run_for_testing():
 
     await async_qa.search(
         query="Find me some vintage shoes under $70",
-        filtering="recall",
         limit=10,
     )
     await async_client.close()
diff --git a/docs/query-agent/guides/search_mode.md b/docs/query-agent/guides/search_mode.md
index d2e87a6b..ff4d6a24 100644
--- a/docs/query-agent/guides/search_mode.md
+++ b/docs/query-agent/guides/search_mode.md
@@ -67,7 +67,7 @@ The `.search()` method accepts several arguments:
 | `query` | `str \| list[ChatMessage]` | The user query you want the agent to search with. This can be a simple string (`"Find me some vintage shoes under $70"`) or a list of chat messages (for conversational context). [See the page on multi-turn conversations for more detail](../reference/multi_turn_conversations.md). |
 | `collections` | `list[str \| QueryAgentCollectionConfig] \| None` | The name(s) of the collections to search. You can pass one or many collection names as a list of strings (e.g., `["ECommerce", "BookSales"]`), or provide collection configuration objects for more control. If specified in the `ask` method, it will overwrite those defined in the instantiation of `QueryAgent`. [See the page on collection configuration for more detail](../reference/advanced_collections.md). |
 | `limit` | `int` | The maximum number of results returned in this page of results. Defaults to `20`. Use [`.next()`](#pagination) to fetch additional pages. |
-| `filtering` | `Literal["recall", "precision"]` | **Required.** Either `"recall"` or `"precision"` to control filter generation. `"recall"` favors more results across filter interpretations; `"precision"` favors strict intent match. Pass `"recall"` for most cases. See [Customized filtering](#customized-filtering) below. |
+| `filtering` | `Literal["recall", "precision"]` | Either `"recall"` or `"precision"` to control filter generation. `"recall"` favors more results across filter interpretations; `"precision"` favors strict intent match. See [Customized filtering](#customized-filtering) below. |
 | `diversity_weight` | `float \| None` | A value between `0.0` and `1.0` that biases the result ranking towards diversity using Maximal Marginal Relevance (MMR). See [Diversity ranking](#diversity-ranking) below. |
 
 </TabItem>
@@ -77,7 +77,7 @@ The `.search()` method accepts several arguments:
 | `query` | `string \| ChatMessage[]` | The user query you want the agent to search with. This can be a simple string (`"Find me some vintage shoes under $70"`) or a list of chat messages (for conversational context). [See the page on multi-turn conversations for more detail](../reference/multi_turn_conversations.md). |
 | `collections` | `(string \| QueryAgentCollectionConfig)[]` | The name(s) of the collections to search. You can pass one or many collection names as a list of strings (e.g., `["ECommerce", "BookSales"]`), or provide collection configuration objects for more control. If specified in the `ask` method, it will overwrite those defined in the instantiation of `QueryAgent`. [See the page on collection configuration for more detail](../reference/advanced_collections.md). |
 | `limit` | `number` | The maximum number of results returned in this page of results. Defaults to `20`. Use [`.next()`](#pagination) to fetch additional pages. |
-| `filtering` | `"recall" \| "precision"` | **Required.** Either `"recall"` or `"precision"` to control filter generation. `"recall"` favors more results across filter interpretations; `"precision"` favors strict intent match. Pass `"recall"` for most cases. See [Customized filtering](#customized-filtering) below. |
+| `filtering` | `"recall" \| "precision"` | Either `"recall"` or `"precision"` to control filter generation. `"recall"` favors more results across filter interpretations; `"precision"` favors strict intent match. See [Customized filtering](#customized-filtering) below. |
 | `diversityWeight` | `number` | A value between `0.0` and `1.0` that biases the result ranking towards diversity using Maximal Marginal Relevance (MMR). See [Diversity ranking](#diversity-ranking) below. |
 
 </TabItem>
@@ -89,9 +89,7 @@ For more advanced searches, you can also specify _additional filters_ within the
 
 Search Mode uses query rewriting to transform your original query into one or multiple Weaviate queries, each with either a search query, metadata filters, or both. The `filtering` parameter controls how many Weaviate queries are generated.
 
-`filtering` is a required argument; pass either `"recall"` or `"precision"`. `"recall"` is recommended when you want more results.
-
-- **`"recall"`** (recommended): Generates multiple Weaviate queries spanning different filters and interpretations of the user query. You should use these when you prefer to get results, even if they don't match every criteria in your query.
+- **`"recall"`** (default): Generates multiple Weaviate queries spanning different filters and interpretations of the user query. You should use these when you prefer to get results, even if they don't match every criteria in your query.
 
 - **`"precision"`**: Generates a single Weaviate query targeting the most likely interpretation of the user query. You should use this when you want the results to follow your query intent closely, even if that means potentially receiving no results.
 

From 4a0924bbc7fd0d8ce6d2d532fa218856cc25fee4 Mon Sep 17 00:00:00 2001
From: Ivan Despot <66276597+g-despot@users.noreply.github.com>
Date: Tue, 7 Jul 2026 12:04:29 +0200
Subject: [PATCH 3/4] test: retry transient external-service failures in
 snippet runners
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The generative/hybrid search snippet tests and the Claude llms.txt fetch test
intermittently red the suite on transient blips (gRPC UNAVAILABLE, deadline
exceeded, read timeouts, 429/5xx) against the shared test cluster + LLM APIs.
Add a narrow transient-error retry (utils.retry_on_transient) around the Python,
TypeScript, and C# snippet runners, and retry the Claude fetch up to 3x. Real
failures (assertion/logic/syntax errors) still fail immediately — only messages
matching a small transient-marker set are retried.
---
 tests/test_csharp.py            | 16 +++++++-
 tests/test_docs_indexability.py | 66 +++++++++++++++++++--------------
 tests/test_python.py            | 23 +++++++-----
 tests/test_typescript.py        |  4 +-
 tests/utils.py                  | 38 +++++++++++++++++++
 5 files changed, 106 insertions(+), 41 deletions(-)

diff --git a/tests/test_csharp.py b/tests/test_csharp.py
index 8c71d8bf..528c314c 100644
--- a/tests/test_csharp.py
+++ b/tests/test_csharp.py
@@ -2,6 +2,8 @@
 import pytest
 import os
 
+import utils
+
 
 CSHARP_CSPROJ = "_includes/code/csharp/WeaviateProject.Tests.csproj"
 
@@ -14,9 +16,19 @@ def run_csharp_test(test_class, empty_weaviates):
     ]
     env = dict(os.environ)
 
+    def _run():
+        result = subprocess.run(command, env=env, capture_output=True, text=True)
+        if result.stdout.strip():
+            print(result.stdout)
+        if result.returncode != 0:
+            raise Exception(
+                f"C# {test_class} failed (exit {result.returncode})\n"
+                f"--- STDERR ---\n{result.stderr}\n--- STDOUT ---\n{result.stdout}"
+            )
+
     try:
-        subprocess.check_call(command, env=env)
-    except subprocess.CalledProcessError as error:
+        utils.retry_on_transient(_run, label=test_class)
+    except Exception as error:
         pytest.fail(f"C# {test_class} failed with error: {error}")
 
 
diff --git a/tests/test_docs_indexability.py b/tests/test_docs_indexability.py
index fc6f00ba..9527020b 100644
--- a/tests/test_docs_indexability.py
+++ b/tests/test_docs_indexability.py
@@ -453,43 +453,53 @@ def test_claude_can_fetch_llms_txt():
     # The llms.txt file starts with "# Weaviate Documentation" and contains
     # section headings like "## agents", "## cloud", "## weaviate".
     # Ask Claude to quote specific content to prove it fetched the real file.
-    response = client.messages.create(
-        model="claude-haiku-4-5-20251001",
-        max_tokens=2048,
-        tools=[{
-            "type": "web_fetch_20250910",
-            "name": "web_fetch",
-            "max_uses": 1,
-            "allowed_domains": ["docs.weaviate.io", "weaviate.io"],
-        }],
-        messages=[{
-            "role": "user",
-            "content": (
-                f"Fetch {url} and tell me: "
-                "1) What is the first heading line of the file (copy it verbatim)? "
-                "2) List ALL the top-level section headings (lines starting with '## '). "
-                "3) Does it mention code examples in multiple languages? Which ones?"
-            ),
-        }],
-    )
+    # LLM responses are non-deterministic, so retry a few times: pass as soon
+    # as one attempt satisfies ALL conditions; only fail if every attempt does.
+    required_sections = ["agents", "cloud", "weaviate"]
+    last_text = ""
+    for attempt in range(3):
+        response = client.messages.create(
+            model="claude-haiku-4-5-20251001",
+            max_tokens=2048,
+            tools=[{
+                "type": "web_fetch_20250910",
+                "name": "web_fetch",
+                "max_uses": 1,
+                "allowed_domains": ["docs.weaviate.io", "weaviate.io"],
+            }],
+            messages=[{
+                "role": "user",
+                "content": (
+                    f"Fetch {url} and tell me: "
+                    "1) What is the first heading line of the file (copy it verbatim)? "
+                    "2) List ALL the top-level section headings (lines starting with '## '). "
+                    "3) Does it mention code examples in multiple languages? Which ones?"
+                ),
+            }],
+        )
+        last_text = _extract_text_from_response(response)
+        tl = last_text.lower()
+        if "weaviate" in tl and all(s in tl for s in required_sections) and "python" in tl:
+            return
+        time.sleep(5)
 
-    text = _extract_text_from_response(response)
-    text_lower = text.lower()
+    # All attempts fell short — surface the last response via the existing assertions
+    tl = last_text.lower()
 
     # Must identify Weaviate
-    assert "weaviate" in text_lower, (
-        f"Claude couldn't identify Weaviate in llms.txt. Response: {text[:500]}"
+    assert "weaviate" in tl, (
+        f"Claude couldn't identify Weaviate in llms.txt. Response: {last_text[:500]}"
     )
 
     # Must find the key top-level sections from llms.txt
-    for section in ["agents", "cloud", "weaviate"]:
-        assert section in text_lower, (
-            f"Claude didn't find '{section}' section in llms.txt. Response: {text[:1000]}"
+    for section in required_sections:
+        assert section in tl, (
+            f"Claude didn't find '{section}' section in llms.txt. Response: {last_text[:1000]}"
         )
 
     # Must identify multi-language code examples
-    assert "python" in text_lower, (
-        f"Claude didn't find Python mentioned in llms.txt. Response: {text[:500]}"
+    assert "python" in tl, (
+        f"Claude didn't find Python mentioned in llms.txt. Response: {last_text[:500]}"
     )
 
 
diff --git a/tests/test_python.py b/tests/test_python.py
index 9d346d8f..478f1517 100644
--- a/tests/test_python.py
+++ b/tests/test_python.py
@@ -4,16 +4,19 @@
 
 
 def run_py_script(script_loc, custom_replace_pairs=None):
-    if custom_replace_pairs:
-        temp_proc_script_loc = utils.load_and_prep_temp_file(
-            script_loc, lang="py", custom_replace_pairs=custom_replace_pairs
-        )
-        utils.execute_py_script_as_module(
-            temp_proc_script_loc.read_text(), Path(script_loc).stem
-        )
-    else:
-        proc_script = utils.load_and_prep_script(script_loc)
-        utils.execute_py_script_as_module(proc_script, Path(script_loc).stem)
+    def _exec():
+        if custom_replace_pairs:
+            temp_proc_script_loc = utils.load_and_prep_temp_file(
+                script_loc, lang="py", custom_replace_pairs=custom_replace_pairs
+            )
+            utils.execute_py_script_as_module(
+                temp_proc_script_loc.read_text(), Path(script_loc).stem
+            )
+        else:
+            proc_script = utils.load_and_prep_script(script_loc)
+            utils.execute_py_script_as_module(proc_script, Path(script_loc).stem)
+
+    utils.retry_on_transient(_exec, label=str(script_loc))
 
 
 def run_pyv3_script(script_loc):
diff --git a/tests/test_typescript.py b/tests/test_typescript.py
index 2f7bf0e3..888de662 100644
--- a/tests/test_typescript.py
+++ b/tests/test_typescript.py
@@ -9,7 +9,9 @@ def run_ts_script(script_loc, custom_replace_pairs=None):
     command = ["npx", "tsx", temp_proc_script_loc]
 
     try:
-        utils.run_script(command, script_loc)
+        utils.retry_on_transient(
+            lambda: utils.run_script(command, script_loc), label=str(script_loc)
+        )
     except Exception as e:
         pytest.fail(str(e))
 
diff --git a/tests/utils.py b/tests/utils.py
index e059bb38..d1ce088e 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -2,12 +2,50 @@
 import re
 import subprocess
 import tempfile
+import time
 import runpy
 from pathlib import Path
 from dotenv import load_dotenv
 
 load_dotenv()
 
+
+# Substrings (case-insensitive) that indicate a transient external-service
+# failure worth retrying (gRPC blips, timeouts, rate limits, 5xx) rather than a
+# real snippet/assertion bug. Kept deliberately narrow.
+TRANSIENT_MARKERS = (
+    "unavailable", "deadline exceeded", "deadline_exceeded",
+    "timed out", "timeout", "read operation timed out",
+    "connection reset", "econnreset", "connection aborted",
+    "502", "503", "504", "429", "too many requests",
+    "rate limit", "overloaded", "temporarily unavailable",
+)
+
+
+def is_transient_error(err) -> bool:
+    """True if the error message looks like a transient external-service blip."""
+    msg = str(err).lower()
+    return any(marker in msg for marker in TRANSIENT_MARKERS)
+
+
+def retry_on_transient(fn, *, retries: int = 2, delay: int = 8, label: str = ""):
+    """Call fn(); on a TRANSIENT error, wait and retry up to `retries` times.
+    Non-transient errors (assertion/logic/syntax) re-raise immediately."""
+    for attempt in range(retries + 1):
+        try:
+            return fn()
+        except Exception as err:
+            if attempt < retries and is_transient_error(err):
+                print(
+                    f"[retry] transient failure on {label} "
+                    f"(attempt {attempt + 1}/{retries + 1}), retrying in {delay}s: "
+                    f"{str(err)[:300]}"
+                )
+                time.sleep(delay)
+                continue
+            raise
+
+
 def load_script(script_path: str) -> str:
     with open(script_path, "r") as f:
         code_block = f.read()

From ec17edb86330f789def2fd39b11cba6cfd0ba233 Mon Sep 17 00:00:00 2001
From: Ivan Despot <66276597+g-despot@users.noreply.github.com>
Date: Tue, 7 Jul 2026 13:09:03 +0200
Subject: [PATCH 4/4] test: harden agents/LLM indexability tests

- Retry transient Weaviate Cloud connection blips in the agents runner (wrap
  execute_py_script_as_module; add connection-error transient markers).
- test_claude_can_fetch_llms_txt: fix stale section assertion (llms.txt no
  longer has 'agents'/'cloud' top-level sections -> check 'quickstart' + 'the
  weaviate stack'), bump max_tokens 2048->4096 to avoid truncated responses.
- test_chatgpt_can_search_code_tabs: web_search paraphrases code, so verify the
  vectorizer is identified (normalized text2vec-weaviate) instead of requiring
  verbatim lines, and drop the vectorizer hint from the prompt so the check
  actually proves the code tabs were read from the site.
---
 tests/test_agents.py            | 10 ++++++++--
 tests/test_docs_indexability.py | 28 ++++++++++++++--------------
 tests/utils.py                  |  2 ++
 3 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/tests/test_agents.py b/tests/test_agents.py
index 13fe8682..5f97d6de 100644
--- a/tests/test_agents.py
+++ b/tests/test_agents.py
@@ -27,7 +27,10 @@
 )
 def test_on_blank_instance_pyv4(script_loc):
     proc_script = utils.load_and_prep_script(script_loc)
-    utils.execute_py_script_as_module(proc_script, Path(script_loc).stem)
+    utils.retry_on_transient(
+        lambda: utils.execute_py_script_as_module(proc_script, Path(script_loc).stem),
+        label=str(script_loc),
+    )
 
 
 @pytest.mark.pyv4
@@ -44,7 +47,10 @@ def test_on_blank_instance_pyv4(script_loc):
 )
 def test_recipes_requiring_openai_pyv4(script_loc):
     proc_script = utils.load_and_prep_script(script_loc)
-    utils.execute_py_script_as_module(proc_script, Path(script_loc).stem)
+    utils.retry_on_transient(
+        lambda: utils.execute_py_script_as_module(proc_script, Path(script_loc).stem),
+        label=str(script_loc),
+    )
 
 
 @pytest.mark.ts
diff --git a/tests/test_docs_indexability.py b/tests/test_docs_indexability.py
index 9527020b..56b3af6c 100644
--- a/tests/test_docs_indexability.py
+++ b/tests/test_docs_indexability.py
@@ -451,16 +451,16 @@ def test_claude_can_fetch_llms_txt():
     url = f"{BASE_URL}/llms.txt"
 
     # The llms.txt file starts with "# Weaviate Documentation" and contains
-    # section headings like "## agents", "## cloud", "## weaviate".
+    # section headings like "## Quickstart", "## The Weaviate stack".
     # Ask Claude to quote specific content to prove it fetched the real file.
     # LLM responses are non-deterministic, so retry a few times: pass as soon
     # as one attempt satisfies ALL conditions; only fail if every attempt does.
-    required_sections = ["agents", "cloud", "weaviate"]
+    required_sections = ["quickstart", "the weaviate stack"]
     last_text = ""
     for attempt in range(3):
         response = client.messages.create(
             model="claude-haiku-4-5-20251001",
-            max_tokens=2048,
+            max_tokens=4096,
             tools=[{
                 "type": "web_fetch_20250910",
                 "name": "web_fetch",
@@ -579,7 +579,7 @@ def test_chatgpt_can_search_code_tabs():
             "Tell me: 1) The exact URL you found "
             "2) What programming languages have code examples "
             "3) For each language, what is the exact vectorizer configuration "
-            "line from the code in the quickstart (e.g. text2vec_weaviate, text2VecWeaviate, etc.)"
+            "line from the code in the quickstart"
         ),
     )
 
@@ -601,16 +601,16 @@ def test_chatgpt_can_search_code_tabs():
         f"Response:\n{text[:1000]}"
     )
 
-    # Must find at least 3 of the 5 exact vectorizer config lines.
-    # web_search_preview may not extract all tabs verbatim, but should
-    # get most of them from the indexed page content.
-    vectorizer_found = sum(
-        1 for line in QUICKSTART_VECTORIZER_LINES.values()
-        if line in text
-    )
-    assert vectorizer_found >= 3, (
-        f"ChatGPT only found {vectorizer_found}/5 vectorizer lines (expected 3+). "
-        f"Response:\n{text[:2000]}"
+    # web_search paraphrases code, so instead of requiring verbatim lines,
+    # confirm ChatGPT identified the text2vec-weaviate vectorizer from the code
+    # tabs (the vectorizer name isn't in the prompt — it can only come from
+    # reading the page). Verbatim per-language extraction is hard-verified
+    # separately by test_claude_can_fetch_code_tabs (direct web_fetch).
+    import re
+    normalized = re.sub(r"[_\-\s]", "", text_lower)
+    assert "text2vecweaviate" in normalized, (
+        f"ChatGPT didn't identify the text2vec-weaviate vectorizer from the "
+        f"quickstart code tabs. Response:\n{text[:2000]}"
     )
 
 
diff --git a/tests/utils.py b/tests/utils.py
index d1ce088e..1ef985b6 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -19,6 +19,8 @@
     "connection reset", "econnreset", "connection aborted",
     "502", "503", "504", "429", "too many requests",
     "rate limit", "overloaded", "temporarily unavailable",
+    "error while connecting", "connection error",
+    "weaviate_cloud_connection_error",
 )