From fd8acb2f8d5efe2276d76c2c58c17504f0a4a5f8 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:56:19 +0530 Subject: [PATCH 1/4] DocumentMAPEvaluator inflates average precision when relevant documents Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- .../components/evaluators/document_map.py | 20 ++++++++++++------- .../fix-document-map-average-precision.yaml | 5 +++++ .../evaluators/test_document_map.py | 20 ++++++++++++++++--- 3 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 releasenotes/notes/fix-document-map-average-precision.yaml diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py index 668ffa4c96f..d46b6f66968 100644 --- a/haystack/components/evaluators/document_map.py +++ b/haystack/components/evaluators/document_map.py @@ -117,19 +117,25 @@ def run( for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents, strict=True): average_precision = 0.0 average_precision_numerator = 0.0 - relevant_documents = 0 - - ground_truth_values = [val for doc in ground_truth if (val := self._get_comparison_value(doc)) is not None] + retrieved_relevant_documents = 0 + + ground_truth_values = [] + for doc in ground_truth: + value = self._get_comparison_value(doc) + if value is not None and value not in ground_truth_values: + ground_truth_values.append(value) + total_relevant_documents = len(ground_truth_values) for rank, retrieved_document in enumerate(retrieved): retrieved_value = self._get_comparison_value(retrieved_document) if retrieved_value is None: continue if retrieved_value in ground_truth_values: - relevant_documents += 1 - average_precision_numerator += relevant_documents / (rank + 1) - if relevant_documents > 0: - average_precision = average_precision_numerator / relevant_documents + ground_truth_values.remove(retrieved_value) + retrieved_relevant_documents += 1 + average_precision_numerator += retrieved_relevant_documents / (rank + 1) + if total_relevant_documents: + average_precision = average_precision_numerator / total_relevant_documents individual_scores.append(average_precision) score = sum(individual_scores) / len(ground_truth_documents) diff --git a/releasenotes/notes/fix-document-map-average-precision.yaml b/releasenotes/notes/fix-document-map-average-precision.yaml new file mode 100644 index 00000000000..67087820f9a --- /dev/null +++ b/releasenotes/notes/fix-document-map-average-precision.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix ``DocumentMAPEvaluator`` to include missed relevant documents in the average precision denominator and avoid + crediting duplicate retrievals of the same document. diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py index fcd54804b6d..62bf5f8031d 100644 --- a/test/components/evaluators/test_document_map.py +++ b/test/components/evaluators/test_document_map.py @@ -95,6 +95,20 @@ def test_run_with_partial_matching(): assert result == {"individual_scores": [1.0, 0.0], "score": 0.5} +@pytest.mark.parametrize( + "retrieved_documents", + [[Document(content="A")], [Document(content="A"), Document(content="A")]], +) +def test_run_with_missed_and_duplicate_relevant_documents(retrieved_documents): + evaluator = DocumentMAPEvaluator() + result = evaluator.run( + ground_truth_documents=[[Document(content="A"), Document(content="B")]], + retrieved_documents=[retrieved_documents], + ) + + assert result == {"individual_scores": [0.5], "score": 0.5} + + def test_run_with_complex_data(): evaluator = DocumentMAPEvaluator() result = evaluator.run( @@ -124,12 +138,12 @@ def test_run_with_complex_data(): "individual_scores": [ 1.0, pytest.approx(0.8333333333333333), - 1.0, + 0.5, pytest.approx(0.5833333333333333), 0.0, - pytest.approx(0.8055555555555555), + pytest.approx(0.8333333333333333), ], - "score": pytest.approx(0.7037037037037037), + "score": pytest.approx(0.625), } From 74a70b6c1dedc6d87a86013cd57043d636d937c0 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:25:20 +0530 Subject: [PATCH 2/4] DocumentMAPEvaluator inflates average precision when relevant documents Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- haystack/components/evaluators/document_map.py | 10 ++++------ .../notes/fix-document-map-average-precision.yaml | 5 +++++ test/components/evaluators/test_document_map.py | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py index d46b6f66968..3f707730261 100644 --- a/haystack/components/evaluators/document_map.py +++ b/haystack/components/evaluators/document_map.py @@ -119,11 +119,9 @@ def run( average_precision_numerator = 0.0 retrieved_relevant_documents = 0 - ground_truth_values = [] - for doc in ground_truth: - value = self._get_comparison_value(doc) - if value is not None and value not in ground_truth_values: - ground_truth_values.append(value) + ground_truth_values = { + value for doc in ground_truth if (value := self._get_comparison_value(doc)) is not None + } total_relevant_documents = len(ground_truth_values) for rank, retrieved_document in enumerate(retrieved): retrieved_value = self._get_comparison_value(retrieved_document) @@ -131,7 +129,7 @@ def run( continue if retrieved_value in ground_truth_values: - ground_truth_values.remove(retrieved_value) + ground_truth_values.discard(retrieved_value) retrieved_relevant_documents += 1 average_precision_numerator += retrieved_relevant_documents / (rank + 1) if total_relevant_documents: diff --git a/releasenotes/notes/fix-document-map-average-precision.yaml b/releasenotes/notes/fix-document-map-average-precision.yaml index 67087820f9a..89f50cd051b 100644 --- a/releasenotes/notes/fix-document-map-average-precision.yaml +++ b/releasenotes/notes/fix-document-map-average-precision.yaml @@ -3,3 +3,8 @@ fixes: - | Fix ``DocumentMAPEvaluator`` to include missed relevant documents in the average precision denominator and avoid crediting duplicate retrievals of the same document. +upgrade: + - | + ``DocumentMAPEvaluator`` scores can change because average precision now uses all unique, valid ground-truth + comparison values as its denominator and credits each value at most once. Re-baseline evaluations that relied on + the previous scores. diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py index 62bf5f8031d..3ae35807a2e 100644 --- a/test/components/evaluators/test_document_map.py +++ b/test/components/evaluators/test_document_map.py @@ -138,10 +138,10 @@ def test_run_with_complex_data(): "individual_scores": [ 1.0, pytest.approx(0.8333333333333333), - 0.5, + 0.5, # Only one of two relevant documents was retrieved. pytest.approx(0.5833333333333333), 0.0, - pytest.approx(0.8333333333333333), + pytest.approx(0.8333333333333333), # The duplicate retrieval is not credited again. ], "score": pytest.approx(0.625), } From 8abd242edf983aac32c14c87e27831f7637a4f3f Mon Sep 17 00:00:00 2001 From: Julian Risch Date: Mon, 10 Aug 2026 13:14:28 +0200 Subject: [PATCH 3/4] style: apply ruff format to DocumentMAPEvaluator test `hatch run fmt-check` was failing on the new parametrize decorator. Co-Authored-By: Claude Opus 5 (1M context) --- test/components/evaluators/test_document_map.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py index 3ae35807a2e..b3aa1d18a08 100644 --- a/test/components/evaluators/test_document_map.py +++ b/test/components/evaluators/test_document_map.py @@ -96,8 +96,7 @@ def test_run_with_partial_matching(): @pytest.mark.parametrize( - "retrieved_documents", - [[Document(content="A")], [Document(content="A"), Document(content="A")]], + "retrieved_documents", [[Document(content="A")], [Document(content="A"), Document(content="A")]] ) def test_run_with_missed_and_duplicate_relevant_documents(retrieved_documents): evaluator = DocumentMAPEvaluator() From c124ae03db4d79a5294a3077c182b94c3f9053e1 Mon Sep 17 00:00:00 2001 From: Julian Risch Date: Mon, 10 Aug 2026 13:16:05 +0200 Subject: [PATCH 4/4] fix: keep DocumentMAPEvaluator working with unhashable comparison values Collecting the ground truth comparison values into a set made the evaluator raise `TypeError: unhashable type` when `document_comparison_field` points to a meta key holding a list or a dict, which worked before. Deduplicate in a list instead, which keeps the equality-based comparison, and drop credited values with `remove`. Co-Authored-By: Claude Opus 5 (1M context) --- haystack/components/evaluators/document_map.py | 17 +++++++++++------ test/components/evaluators/test_document_map.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py index 3f707730261..eb5180fe909 100644 --- a/haystack/components/evaluators/document_map.py +++ b/haystack/components/evaluators/document_map.py @@ -119,17 +119,22 @@ def run( average_precision_numerator = 0.0 retrieved_relevant_documents = 0 - ground_truth_values = { - value for doc in ground_truth if (value := self._get_comparison_value(doc)) is not None - } - total_relevant_documents = len(ground_truth_values) + # A list keeps the deduplication working for unhashable comparison values, for example when + # document_comparison_field points to a meta key holding a list. + uncredited_ground_truth_values: list[Any] = [] + for doc in ground_truth: + value = self._get_comparison_value(doc) + if value is not None and value not in uncredited_ground_truth_values: + uncredited_ground_truth_values.append(value) + + total_relevant_documents = len(uncredited_ground_truth_values) for rank, retrieved_document in enumerate(retrieved): retrieved_value = self._get_comparison_value(retrieved_document) if retrieved_value is None: continue - if retrieved_value in ground_truth_values: - ground_truth_values.discard(retrieved_value) + if retrieved_value in uncredited_ground_truth_values: + uncredited_ground_truth_values.remove(retrieved_value) retrieved_relevant_documents += 1 average_precision_numerator += retrieved_relevant_documents / (rank + 1) if total_relevant_documents: diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py index b3aa1d18a08..07c1288a531 100644 --- a/test/components/evaluators/test_document_map.py +++ b/test/components/evaluators/test_document_map.py @@ -65,6 +65,18 @@ def test_run_with_nested_meta_comparison(): assert result == {"individual_scores": [1.0, 0.0], "score": 0.5} +def test_run_with_unhashable_meta_comparison(): + evaluator = DocumentMAPEvaluator(document_comparison_field="meta.tags") + result = evaluator.run( + ground_truth_documents=[ + [Document(content="x", meta={"tags": ["a"]}), Document(content="y", meta={"tags": ["b"]})] + ], + retrieved_documents=[[Document(content="z", meta={"tags": ["a"]})]], + ) + + assert result == {"individual_scores": [0.5], "score": 0.5} + + def test_run_with_all_matching(): evaluator = DocumentMAPEvaluator() result = evaluator.run(