Ninadnj · Ninadnj · Aug 12, 2026 · Aug 11, 2026 · Aug 12, 2026
diff --git a/README.md b/README.md
diff --git a/eval/results.json b/eval/results.json
@@ -207,6 +207,24 @@
       "labelled_recall": 0.929,
       "off_topic_memories_returned": 2,
       "off_topic_max": 12
+    },
+    {
+      "min_score": 0.3,
+      "labelled_recall": 0.714,
+      "off_topic_memories_returned": 1,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.35,
+      "labelled_recall": 0.571,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.4,
+      "labelled_recall": 0.5,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
     }
   ]
 }
diff --git a/eval/results.md b/eval/results.md
@@ -44,11 +44,14 @@ Distractor memories are added from the same project. The baseline grows linearly
 | 0.15 | 0.93 | 6/12 |
 | 0.20 | 0.93 | 2/12 |
 | 0.25 | 0.93 | 2/12 |
+| 0.30 | 0.71 | 1/12 |
+| 0.35 | 0.57 | 0/12 |
+| 0.40 | 0.50 | 0/12 |
 
 Four deliberately off-topic queries stand in for a task the store knows nothing about. Without a floor the engine returns k memories anyway, and the MCP layer strips scores, so the agent cannot tell. `HashingEmbedder.recommended_min_score` is set from this sweep.
 
 ## Honest limits
 
 - 7 tasks and 10 gold labels, all hand-written by the author. This is an engineering check, not a production-scale claim; one retrieval either way moves recall by ~0.07.
 - The labels, the queries and the retriever all come from the same person, which is exactly the setup that flatters a retriever. The paraphrase arm exists to push back on that.
-- Numbers use the offline hashing embedder so they are stable in CI. The `sentence-transformers` backend is not measured here.
+- Numbers use the offline hashing embedder so they are stable in CI. The `sentence-transformers` backend is measured separately in `results_sentence_transformers.md`.
diff --git a/eval/results_sentence_transformers.json b/eval/results_sentence_transformers.json
@@ -0,0 +1,230 @@
+{
+  "k": 3,
+  "budget": 120,
+  "seed": 0,
+  "exact_tokenizer": true,
+  "embedder": "SentenceTransformerEmbedder",
+  "n_memories": 14,
+  "n_tasks": 7,
+  "summary": {
+    "no_memory": {
+      "avg_context_tokens": 0.0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "token_reduction_vs_baseline": 100.0
+    },
+    "full_context": {
+      "avg_context_tokens": 424.0,
+      "precision": 0.102,
+      "recall": 1.0,
+      "token_reduction_vs_baseline": 0.0
+    },
+    "random_k": {
+      "avg_context_tokens": 93.4,
+      "precision": 0.048,
+      "recall": 0.071,
+      "token_reduction_vs_baseline": 78.0
+    },
+    "retrieval": {
+      "avg_context_tokens": 93.4,
+      "precision": 0.381,
+      "recall": 0.857,
+      "token_reduction_vs_baseline": 78.0
+    },
+    "budget_recall": {
+      "avg_context_tokens": 93.4,
+      "precision": 0.381,
+      "recall": 0.857,
+      "token_reduction_vs_baseline": 78.0
+    }
+  },
+  "per_task": [
+    {
+      "task": "task_1",
+      "query": "Fix the bug where customers receive two confirmation emails for a single booking.",
+      "gold": [
+        "mem_0005",
+        "mem_0013"
+      ],
+      "retrieved": [
+        "mem_0005",
+        "mem_0014",
+        "mem_0008"
+      ],
+      "recall": 0.5,
+      "tokens": 94
+    },
+    {
+      "task": "task_2",
+      "query": "Add rate limiting to the chat endpoint so the Gemini API key cannot be abused.",
+      "gold": [
+        "mem_0004",
+        "mem_0011"
+      ],
+      "retrieved": [
+        "mem_0011",
+        "mem_0004",
+        "mem_0008"
+      ],
+      "recall": 1.0,
+      "tokens": 96
+    },
+    {
+      "task": "task_3",
+      "query": "A booking shows the wrong time to the customer. Investigate how timezones are handled for bookings.",
+      "gold": [
+        "mem_0003"
+      ],
+      "retrieved": [
+        "mem_0003",
+        "mem_0005",
+        "mem_0014"
+      ],
+      "recall": 1.0,
+      "tokens": 92
+    },
+    {
+      "task": "task_4",
+      "query": "Update the list of services and prices the chatbot tells customers about.",
+      "gold": [
+        "mem_0004",
+        "mem_0012"
+      ],
+      "retrieved": [
+        "mem_0004",
+        "mem_0011",
+        "mem_0010"
+      ],
+      "recall": 0.5,
+      "tokens": 101
+    },
+    {
+      "task": "task_5",
+      "query": "Make the gallery images publicly viewable while keeping all other uploads private.",
+      "gold": [
+        "mem_0007"
+      ],
+      "retrieved": [
+        "mem_0007",
+        "mem_0011",
+        "mem_0014"
+      ],
+      "recall": 1.0,
+      "tokens": 95
+    },
+    {
+      "task": "task_6",
+      "query": "Add a new admin-only settings page and make sure it is protected like the other admin routes.",
+      "gold": [
+        "mem_0006"
+      ],
+      "retrieved": [
+        "mem_0006",
+        "mem_0012",
+        "mem_0007"
+      ],
+      "recall": 1.0,
+      "tokens": 80
+    },
+    {
+      "task": "task_7",
+      "query": "The chat widget sometimes crashes when Gemini replies. Make the chat resilient to bad responses.",
+      "gold": [
+        "mem_0008"
+      ],
+      "retrieved": [
+        "mem_0008",
+        "mem_0011",
+        "mem_0004"
+      ],
+      "recall": 1.0,
+      "tokens": 96
+    }
+  ],
+  "phrasing": {
+    "query": {
+      "recall": 0.857,
+      "avg_query_word_overlap_with_gold": 0.403
+    },
+    "paraphrase": {
+      "recall": 0.786,
+      "avg_query_word_overlap_with_gold": 0.032
+    }
+  },
+  "scaling": [
+    {
+      "n_memories": 14,
+      "full_context_tokens": 424,
+      "budget_recall_tokens": 93.4,
+      "budget_recall": 0.857
+    },
+    {
+      "n_memories": 34,
+      "full_context_tokens": 805,
+      "budget_recall_tokens": 87.3,
+      "budget_recall": 0.857
+    },
+    {
+      "n_memories": 54,
+      "full_context_tokens": 1159,
+      "budget_recall_tokens": 86.3,
+      "budget_recall": 0.857
+    }
+  ],
+  "min_score_sweep": [
+    {
+      "min_score": 0.0,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 12,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.05,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 10,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.1,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 1,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.15,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.2,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.25,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.3,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.35,
+      "labelled_recall": 0.857,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
+    },
+    {
+      "min_score": 0.4,
+      "labelled_recall": 0.714,
+      "off_topic_memories_returned": 0,
+      "off_topic_max": 12
+    }
+  ]
+}
diff --git a/eval/results_sentence_transformers.md b/eval/results_sentence_transformers.md
@@ -0,0 +1,57 @@
+# Evaluation results
+
+- Benchmark: **14 memories**, **7 tasks**, top-k = **3**, budget = **120 tokens**
+- Embedder: `SentenceTransformerEmbedder` · exact tokenizer: `True`
+
+| Arm | Avg context tokens | Recall | Precision | Tokens saved vs baseline |
+| --- | ---: | ---: | ---: | ---: |
+| No memory (control) | 0 | 0.00 | 0.00 | 100% |
+| Full context (baseline — load every memory) | 424 | 1.00 | 0.10 | 0% |
+| Random k (control — the saving without the retrieval) | 93 | 0.07 | 0.05 | 78% |
+| Targeted retrieval (this engine) | 93 | 0.86 | 0.38 | 78% |
+| Budget recall (≤ 120 tokens) | 93 | 0.86 | 0.38 | 78% |
+
+**Read the random arm first.** It loads the same number of memories as the engine, so it reports the same ~78% token saving — at **0.07** recall against the engine's **0.86**. The saving is arithmetic (k memories out of n); only the recall gap is evidence that retrieval does anything.
+
+Precision is reported for completeness, but the baseline's **0.10** is just `|relevant| / |store|` — an artefact of loading everything, not a meaningful comparison.
+
+## Does it survive a rephrase?
+
+| Query phrasing | Word overlap with gold memories | Recall |
+| --- | ---: | ---: |
+| Developer phrasing (as labelled) | 40% | 0.86 |
+| Outsider paraphrase (vocabulary avoided) | 3% | 0.79 |
+
+This backend embeds meaning rather than wording, so the drop is only **0.07**. It is the reason to install the `real` extra for day-to-day use: real questions rarely reuse the words a memory was written in.
+
+## Does the cost stay flat as the store grows?
+
+| Memories in store | Full context tokens | Budget recall tokens | Budget recall |
+| ---: | ---: | ---: | ---: |
+| 14 | 424 | 93 | 0.86 |
+| 34 | 805 | 87 | 0.86 |
+| 54 | 1159 | 86 | 0.86 |
+
+Distractor memories are added from the same project. The baseline grows linearly; the budgeted arm does not. Recall is measured against the same labels throughout, so any drop is real interference from the added memories.
+
+## Relevance floor (`min_score`)
+
+| min_score | Labelled recall | Off-topic memories returned |
+| ---: | ---: | ---: |
+| 0.00 | 0.86 | 12/12 |
+| 0.05 | 0.86 | 10/12 |
+| 0.10 | 0.86 | 1/12 |
+| 0.15 | 0.86 | 0/12 |
+| 0.20 | 0.86 | 0/12 |
+| 0.25 | 0.86 | 0/12 |
+| 0.30 | 0.86 | 0/12 |
+| 0.35 | 0.86 | 0/12 |
+| 0.40 | 0.71 | 0/12 |
+
+Four deliberately off-topic queries stand in for a task the store knows nothing about. Without a floor the engine returns k memories anyway, and the MCP layer strips scores, so the agent cannot tell. `SentenceTransformerEmbedder.recommended_min_score` is set from this sweep.
+
+## Honest limits
+
+- 7 tasks and 10 gold labels, all hand-written by the author. This is an engineering check, not a production-scale claim; one retrieval either way moves recall by ~0.07.
+- The labels, the queries and the retriever all come from the same person, which is exactly the setup that flatters a retriever. The paraphrase arm exists to push back on that.
+- Numbers use `sentence-transformers` (all-MiniLM-L6-v2), which needs a model download and is therefore not run in CI. Regenerate with: `python eval/run_eval.py --embedder sentence-transformers`.