diff --git a/dingo/model/llm/llm_search_result_authority.py b/dingo/model/llm/llm_search_result_authority.py new file mode 100644 index 00000000..36f9ec42 --- /dev/null +++ b/dingo/model/llm/llm_search_result_authority.py @@ -0,0 +1,322 @@ +"""Rule-based search result authority grader.""" + +from __future__ import annotations +import html +import math +import re +import statistics +from dataclasses import dataclass +from typing import Any + +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model import Model + +PRESTIGIOUS_VENUE_PATTERNS = ( + r"^nature(?:$|\s)", + r"^npj(?:$|\s)", + r"^communications (?:biology|chemistry|earth & environment|materials|physics)$", + r"^scientific (?:reports|data)$", + r"^science$", + r"^science (?:advances|immunology|robotics|signaling|translational medicine)$", + r"^cell$", + r"^cell (?:reports|metabolism|systems|stem cell|chemical biology|host & microbe|genomics)$", + r"^(?:cancer|molecular|developmental) cell$", + r"^(?:the )?lancet(?:$|\s)", + r"^(?:the )?new england journal of medicine$", + r"^nejm(?:$|\s)", + r"^jama(?:$|\s)", + r"^(?:the )?bmj$", + r"^(?:proceedings of the national academy of sciences(?: of the united states of america)?|pnas)$", + r"^journal of the american chemical society$", + r"^physical review letters$", + r"^angewandte chemie(?: international edition)?$", + r"^(?:neurips|icml|iclr|cvpr|acl|emnlp|aaai|ijcai|sigir)$", + r"^advances in neural information processing systems$", +) + +RECOGNIZED_VENUE_PATTERNS = ( + r"\bieee\b", + r"\bacm\b", + r"^plos(?:$|\s)", + r"^frontiers in ", + r"^bmj(?:$|\s)", +) + +RECOGNIZED_PUBLISHER_HINTS = ( + "nature portfolio", + "springer nature", + "springer", + "elsevier", + "wiley", + "taylor & francis", + "routledge", + "sage publishing", + "oxford university press", + "cambridge university press", + "institute of electrical and electronics engineers", + "association for computing machinery", + "american chemical society", + "royal society of chemistry", + "institute of physics", + "iop publishing", + "american physical society", + "american institute of physics", + "frontiers media", + "public library of science", + "bmj publishing", + "wolters kluwer", + "lippincott williams & wilkins", + "american geophysical union", + "royal society", + "american association for the advancement of science", + "de gruyter", + "crc press", + "chapman & hall", + "world scientific", + "academic press", + "humana press", + "emerald publishing", +) + +REPOSITORY_VENUE_HINTS = ( + "arxiv", + "biorxiv", + "medrxiv", + "chemrxiv", + "ssrn", + "zenodo", + "figshare", + "mendeley data", + "open science framework", + "osf preprints", + "research square", + "repository", +) + + +def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float: + return max(low, min(high, value)) + + +def _normalize_text(text: Any) -> str: + value = html.unescape(str(text or "")) + value = re.sub(r"<[^>]+>", " ", value) + return " ".join(value.lower().split()) + + +def _venue_aliases(venue: str) -> list[str]: + aliases = [part.strip(" .,:;-") for part in venue.split("|") if part.strip(" .,:;-")] + return aliases or [venue] + + +def _matches_venue_patterns(venue: str, patterns: tuple[str, ...]) -> bool: + return any(re.search(pattern, alias) for alias in _venue_aliases(venue) for pattern in patterns) + + +def _extract_publishers(result: dict[str, Any]) -> str: + value = result.get("publication_publisher") or result.get("publisher") or "" + if isinstance(value, list): + return _normalize_text(" | ".join(str(item) for item in value if item)) + return _normalize_text(value) + + +def _has_venue_issn(result: dict[str, Any]) -> bool: + value = result.get("publication_venue_issn") or result.get("issn") or [] + values = value if isinstance(value, list) else [value] + return any(re.fullmatch(r"\d{4}-?\d{3}[\dXx]", str(item or "").strip()) for item in values) + + +def extract_venue(result: dict[str, Any]) -> str: + return str( + result.get("publication_venue_name_unified") + or result.get("publication_venue_name") + or result.get("venue") + or result.get("source") + or "" + ) + + +def extract_citations(result: dict[str, Any], key: str = "citation_count") -> float: + try: + return float(result.get(key) or 0) + except (TypeError, ValueError): + return 0.0 + + +def _has_doi_in_locations(locations: Any) -> bool: + """Return whether a valid locations list contains a DOI URL or value.""" + if not isinstance(locations, list): + return False + + for location in locations: + if isinstance(location, dict): + values = ( + location.get("doi"), + location.get("url"), + location.get("landing_page_url"), + ) + elif isinstance(location, str): + values = (location,) + else: + continue + + if any("doi.org/" in str(value or "").lower() for value in values): + return True + return False + + +@dataclass +class AuthorityGrade: + """Structured authority score for one search result.""" + + score: float = 0.0 + citation_score: float = 0.0 + influential_citation_score: float = 0.0 + venue_score: float = 0.0 + doi_score: float = 0.0 + reason: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "score": round(self.score, 5), + "citation_score": round(self.citation_score, 5), + "influential_citation_score": round(self.influential_citation_score, 5), + "venue_score": round(self.venue_score, 5), + "doi_score": round(self.doi_score, 5), + "reason": self.reason, + } + + +@dataclass +class AuthoritySummary: + mean_score: float = 0.0 + median_score: float = 0.0 + mean_citation_score: float = 0.0 + mean_influential_citation_score: float = 0.0 + mean_venue_score: float = 0.0 + mean_doi_score: float = 0.0 + graded_pairs: int = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "authority_mean_score": round(self.mean_score, 5), + "authority_median_score": round(self.median_score, 5), + "authority_mean_citation_score": round(self.mean_citation_score, 5), + "authority_mean_influential_citation_score": round(self.mean_influential_citation_score, 5), + "authority_mean_venue_score": round(self.mean_venue_score, 5), + "authority_mean_doi_score": round(self.mean_doi_score, 5), + "authority_graded_pairs": self.graded_pairs, + } + + +@Model.llm_register("LLMSearchResultAuthority") +class LLMSearchResultAuthority: + """Authority scorer based on citation impact, venue, and DOI metadata.""" + + dynamic_config = EvaluatorLLMArgs() + default_threshold = 0.15 + + def grade(self, *, result: dict[str, Any]) -> AuthorityGrade: + venue = _normalize_text(extract_venue(result)) + venue_type = _normalize_text(result.get("publication_venue_type") or "") + publishers = _extract_publishers(result) + citations = extract_citations(result, "citation_count") + influential = extract_citations(result, "influential_citation_count") + + citation_score = _clamp(math.log1p(citations) / math.log1p(500.0)) + influential_score = _clamp(math.log1p(influential) / math.log1p(50.0)) + + venue_score = 0.25 + reason = "unknown_or_low_signal_venue" + is_repository = "repository" in venue_type or "preprint" in venue_type or any( + hint in venue for hint in REPOSITORY_VENUE_HINTS + ) + is_academic_book = "book series" in venue_type or "ebook platform" in venue_type or "ebooks" in venue + if is_repository: + venue_score = 0.45 + reason = "repository_or_preprint" + elif is_academic_book: + venue_score = 0.55 + reason = "academic_book_series" + elif _matches_venue_patterns(venue, PRESTIGIOUS_VENUE_PATTERNS): + venue_score = 0.85 + reason = "prestigious_venue_family" + elif _matches_venue_patterns(venue, RECOGNIZED_VENUE_PATTERNS) or any( + hint in publishers for hint in RECOGNIZED_PUBLISHER_HINTS + ): + venue_score = 0.75 + reason = "recognized_scholarly_publisher_or_venue" + elif "journal" in venue_type or "conference" in venue_type or _has_venue_issn(result): + venue_score = 0.65 + reason = "structured_journal_or_conference" + elif venue: + venue_score = 0.40 + reason = "named_venue" + + doi_score = 1.0 if result.get("doi") or _has_doi_in_locations(result.get("locations")) else 0.0 + score = ( + 0.45 * citation_score + + 0.20 * influential_score + + 0.25 * venue_score + + 0.10 * doi_score + ) + return AuthorityGrade( + score=_clamp(score), + citation_score=citation_score, + influential_citation_score=influential_score, + venue_score=venue_score, + doi_score=doi_score, + reason=reason, + ) + + @classmethod + def _config_value(cls, name: str, default: Any = None) -> Any: + return getattr(cls.dynamic_config, name, default) + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Executor entry point: evaluate one flattened search result row.""" + result = getattr(input_data, "search_result", None) + if not isinstance(result, dict): + result = input_data.to_dict() + + grade = cls().grade(result=result) + threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold) + + labels: list[str] = [] + if grade.score < threshold: + labels.append("Authority.Error_Authority_Low") + if grade.citation_score <= 0.0: + labels.append("Authority.Error_Citation_Miss") + if grade.venue_score <= 0.25: + labels.append("Authority.Error_Venue_Low_Signal") + if grade.doi_score <= 0.0: + labels.append("Authority.Error_DOI_Miss") + + status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + + return EvalDetail( + metric=cls.__name__, + status=status, + score=round(grade.score, 5), + label=labels, + reason=[grade.to_dict()], + ) + + +def aggregate_grades(grades: list[AuthorityGrade]) -> AuthoritySummary: + if not grades: + return AuthoritySummary() + return AuthoritySummary( + mean_score=statistics.mean(g.score for g in grades), + median_score=statistics.median(g.score for g in grades), + mean_citation_score=statistics.mean(g.citation_score for g in grades), + mean_influential_citation_score=statistics.mean(g.influential_citation_score for g in grades), + mean_venue_score=statistics.mean(g.venue_score for g in grades), + mean_doi_score=statistics.mean(g.doi_score for g in grades), + graded_pairs=len(grades), + ) diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py new file mode 100644 index 00000000..e1da7af2 --- /dev/null +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -0,0 +1,782 @@ +"""Search result effectiveness grader. + +This grader scores whether a returned search result has enough usable +bibliographic content for a user to judge and consume it. Missing-field and +basic information-density checks are deterministic. Readability and corruption +checks can be delegated to an LLM judge to avoid over-penalizing normal academic +formulas, units, and symbols. + +It intentionally does not judge topical relevance; use +``LLMSearchResultRelevance`` for that. +""" + +from __future__ import annotations +import json +import logging +import re +import statistics +import time +from dataclasses import dataclass +from typing import Any + +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model import Model + +logger = logging.getLogger(__name__) + + +RULE_SPECIAL_CHARACTER_PATTERNS = ( + r"u200e", + r"÷|\? :", + r"[锟解枴閿熻В鏋碷�]|\{\/U\}", + r"U\+26[0-F][0-D]|U\+273[3-4]|U\+1F[3-6][0-4][0-F]|U\+1F6[8-F][0-F]", + r"<\|.*?\|>", + r"<[^>]+>", +) +RULE_INVISIBLE_CHAR_PATTERN = r"[\u0080-\u009F\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]" +RULE_ABNORMAL_CHAR_THRESHOLD = 0.01 +HTML_TAG_PATTERN = r"<[^>]+>" +MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}" +UTF8_LATIN1_SEQUENCE_PATTERN = re.compile(r"[\u00C2\u00C3\u00D0\u00D1][\u0080-\u00BF]") +C1_CONTROL_PATTERN = re.compile(r"[\u0080-\u009F]") +UNICODE_REPLACEMENT_CHARACTER = "\ufffd" + + +LLM_FIELD_QUALITY_SYSTEM_PROMPT = """You are a strict but practical data quality evaluator for +academic search result metadata. + +Judge whether each supplied metadata field is readable and clean enough to show to users. +Focus on real text-quality problems: +- missing or empty field +- invisible/control characters +- mojibake or garbled encoding, such as replacement characters, unreadable CJK mojibake, + or UTF-8 text decoded as Latin-1 with repeated sequences like Ð... or Ñ... +- raw HTML/XML markup leaked into visible text, such as ... +- suspicious special-character noise that materially hurts readability + +Do NOT penalize normal academic content: +- mathematical formulas, LaTeX, chemical symbols, units, Greek letters +- punctuation, pipes used as separators, parentheses, slashes, hyphens +- mixed Chinese/English titles, journal names, abbreviations, DOI-like text + +Return compact JSON only. Do not use markdown. Keep each reason within 12 words +and do not use double quotes inside reasons. +Schema: +{ + "fields": { + "title": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "abstract": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "keywords": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "venue": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "author": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."} + }, + "overall_issues": ["..."], + "reason": "short overall reason" +} + +Use issue names from: +- missing_field +- invisible_char +- mojibake +- html_tag +- unreadable_text +- special_char_noise +- none + +Scoring guidance: +- 1.0: clean, readable field; normal formulas and units are allowed. +- 0.7: mostly readable with minor display artifacts. +- 0.4: readable but contains visible markup or notable noise requiring cleanup. +- 0.1: unreadable garbled text, heavy mojibake, or severe invisible/control-character corruption. +- 0.0: missing/empty field. +""" + + +def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float: + return max(low, min(high, value)) + + +def _presence_quality(value: Any) -> float: + """Score field presence without using content length as a quality proxy.""" + return 1.0 if str(value or "").strip() else 0.0 + + +def _suspicious_latin1_mojibake_count(text: str) -> int: + return len(UTF8_LATIN1_SEQUENCE_PATTERN.findall(text)) + len(C1_CONTROL_PATTERN.findall(text)) + + +def _looks_like_utf8_latin1_mojibake(text: str) -> bool: + """Detect UTF-8 text accidentally decoded as Latin-1. + + Literal characters such as ``Ð`` and ``Ñ`` can be valid text, so they are + not sufficient evidence by themselves. A value is treated as suspicious + when it contains repeated UTF-8/Latin-1 byte-shaped sequences or C1 control + characters and a reversible Latin-1-to-UTF-8 repair removes that evidence. + """ + value = str(text or "") + if not value: + return False + + suspicious_before = _suspicious_latin1_mojibake_count(value) + if suspicious_before == 0: + return False + + c1_count = len(C1_CONTROL_PATTERN.findall(value)) + repeated_sequences = len(UTF8_LATIN1_SEQUENCE_PATTERN.findall(value)) >= 2 + if not repeated_sequences and c1_count / max(1, len(value)) < RULE_ABNORMAL_CHAR_THRESHOLD: + return False + + try: + repaired = value.encode("latin-1").decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + # Mixed-language metadata may contain legitimate non-Latin-1 text next + # to a corrupted fragment. Repeated byte-shaped pairs plus C1 controls + # are enough to send that field to the LLM judge for confirmation. + return repeated_sequences and c1_count > 0 + + suspicious_after = _suspicious_latin1_mojibake_count(repaired) + return repaired != value and suspicious_after < suspicious_before + + +def _has_mojibake_evidence(text: str) -> bool: + value = str(text or "") + return ( + UNICODE_REPLACEMENT_CHARACTER in value + or bool(re.search(MOJIBAKE_EVIDENCE_PATTERN, value)) + or _looks_like_utf8_latin1_mojibake(value) + ) + + +def _rule_abnormal_char_issues(text: str) -> list[str]: + value = str(text or "") + if not value: + return [] + + issues: list[str] = [] + special_matches: list[str] = [] + for pattern in RULE_SPECIAL_CHARACTER_PATTERNS: + special_matches.extend(re.findall(pattern, value)) + has_html_tag = bool(re.search(HTML_TAG_PATTERN, value)) + if has_html_tag or len(special_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: + issues.append("RuleSpecialCharacter") + + has_mojibake = _has_mojibake_evidence(value) + if has_mojibake: + issues.append("RuleMojibake") + + invisible_matches = re.findall(RULE_INVISIBLE_CHAR_PATTERN, value) + if not has_mojibake and len(invisible_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: + issues.append("RuleInvisibleChar") + return issues + + +def _has_confirmed_llm_issue(issues: list[str] | None) -> bool: + if not issues: + return False + ignored = {"none", "missing_field"} + return any(str(issue).split(":")[-1].strip().lower() not in ignored for issue in issues) + + +def _filter_llm_field_issues(field: str, value: str, issues: list[str]) -> list[str]: + """Keep only LLM issues supported by field-level evidence.""" + text = str(value or "") + filtered: list[str] = [] + for issue in issues: + issue_type = str(issue).split(":")[-1].strip().lower() + keep = False + if issue_type == "html_tag": + keep = bool(re.search(HTML_TAG_PATTERN, text)) + elif issue_type == "invisible_char": + keep = bool(re.search(RULE_INVISIBLE_CHAR_PATTERN, text)) + elif issue_type in {"mojibake", "unreadable_text"}: + keep = _has_mojibake_evidence(text) + elif issue_type == "special_char_noise": + keep = bool(_rule_abnormal_char_issues(text)) + else: + keep = True + + if keep and issue not in filtered: + filtered.append(issue) + return filtered + + +def _strip_json_fence(text: str) -> str: + value = (text or "").strip() + if value.startswith("```json"): + value = value[7:].strip() + elif value.startswith("```"): + value = value[3:].strip() + if value.endswith("```"): + value = value[:-3].strip() + return value + + +def _extract_json_object(text: str) -> str: + value = _strip_json_fence(text) + start = value.find("{") + end = value.rfind("}") + if start >= 0 and end > start: + return value[start:end + 1] + return value + + +def _safe_float(value: Any, default: float = 1.0) -> float: + try: + return _clamp(float(value)) + except (TypeError, ValueError): + return default + + +def _normalize_issues(value: Any) -> list[str]: + if not value: + return [] + if isinstance(value, str): + return [] if value.lower() == "none" else [value] + if isinstance(value, list): + issues = [] + for item in value: + item_text = str(item).strip() + if item_text and item_text.lower() != "none": + issues.append(item_text) + return issues + return [str(value)] + + +EFFECTIVENESS_LABEL_MAP = { + "missing_title": "Effectiveness.Error_Title_Miss", + "missing_abstract": "Effectiveness.Error_Abstract_Miss", + "missing_keywords": "Effectiveness.Error_Keywords_Miss", + "missing_author": "Effectiveness.Error_Author_Miss", + "html_tag": "Effectiveness.Error_HTML_Tag", + "mojibake": "Effectiveness.Error_Mojibake", + "invisible_char": "Effectiveness.Error_Invisible_Char", + "unreadable_text": "Effectiveness.Error_Unreadable_Text", + "special_char_noise": "Effectiveness.Error_Special_Char_Noise", + "llm_quality_parse_error": "Effectiveness.Error_LLM_Quality_Parse", + "RuleSpecialCharacter": "Effectiveness.Error_Rule_Special_Character", + "RuleInvisibleChar": "Effectiveness.Error_Rule_Invisible_Char", + "RuleMojibake": "Effectiveness.Error_Mojibake", +} + + +def _issue_to_label(issue: str) -> str | None: + issue_text = str(issue or "").strip() + if not issue_text: + return None + issue_type = issue_text.split(":")[-1] + return EFFECTIVENESS_LABEL_MAP.get(issue_type) + + +def _issues_to_labels(issues: list[str] | None) -> list[str]: + """Map issues to final business labels. + + RuleSpecialCharacter and RuleInvisibleChar are candidate triggers. When LLM + confirms a concrete issue such as title:html_tag, keep the concrete + business label and suppress the intermediate rule label to avoid duplicate + output files for the same problem. + """ + labels: list[str] = [] + normalized_issues = [str(issue or "").strip() for issue in (issues or []) if str(issue or "").strip()] + has_confirmed_quality_issue = any( + ":" in issue and _issue_to_label(issue) is not None + for issue in normalized_issues + ) + + for issue in normalized_issues: + issue_type = issue.split(":")[-1] + if has_confirmed_quality_issue and issue_type in { + "RuleSpecialCharacter", + "RuleInvisibleChar", + "RuleMojibake", + }: + continue + label = _issue_to_label(issue) + if label and label not in labels: + labels.append(label) + return labels + + +def _truncate_for_llm(value: str, max_chars: int = 1000) -> str: + text = str(value or "") + if len(text) <= max_chars: + return text + return text[:max_chars] + "...[truncated]" + + +def extract_keywords(result: dict[str, Any]) -> list[str]: + value = result.get("keywords") or result.get("keyword") or result.get("concepts") or [] + if isinstance(value, str): + return [item.strip() for item in re.split(r"[,;|]", value) if item.strip()] + if isinstance(value, list): + keywords: list[str] = [] + for item in value: + if isinstance(item, dict): + name = item.get("name") or item.get("display_name") or item.get("keyword") + if name: + keywords.append(str(name)) + elif item not in (None, ""): + keywords.append(str(item)) + return keywords + return [] + + +def extract_venue(result: dict[str, Any]) -> str: + return str( + result.get("publication_venue_name_unified") + or result.get("publication_venue_name") + or result.get("venue") + or result.get("source") + or "" + ) + + +def extract_authors(result: dict[str, Any]) -> list[str]: + """Extract author names from common search API response shapes.""" + value = result.get("author") or result.get("authors") or [] + if isinstance(value, str): + return [item.strip() for item in re.split(r"[;|]", value) if item.strip()] + if isinstance(value, dict): + value = [value] + if not isinstance(value, list): + return [] + + authors: list[str] = [] + for item in value: + if isinstance(item, dict): + name = item.get("name") or item.get("display_name") or item.get("author_name") + if name: + authors.append(str(name).strip()) + elif item not in (None, ""): + authors.append(str(item).strip()) + return [author for author in authors if author] + + +@dataclass +class LLMFieldQuality: + """LLM readability and corruption judgment for one search result.""" + + title_score: float = 1.0 + abstract_score: float = 1.0 + keywords_score: float = 1.0 + venue_score: float = 1.0 + author_score: float = 1.0 + issues: list[str] | None = None + reason: str = "" + error: str = "" + + def field_score(self, field: str) -> float: + return { + "title": self.title_score, + "abstract": self.abstract_score, + "keywords": self.keywords_score, + "venue": self.venue_score, + "author": self.author_score, + }.get(field, 1.0) + + +def _parse_llm_field_quality_response(text: str) -> LLMFieldQuality: + candidate = _extract_json_object(text) + try: + data = json.loads(candidate) + except json.JSONDecodeError as e: + return LLMFieldQuality(error=f"JSON parse failed: {e}. Text: {text[:200]}") + + fields = data.get("fields") if isinstance(data, dict) else {} + if not isinstance(fields, dict): + return LLMFieldQuality(error=f"Missing fields object. Text: {candidate[:200]}") + + issues: list[str] = [] + scores: dict[str, float] = {} + reasons: list[str] = [] + for field in ("title", "abstract", "keywords", "venue", "author"): + field_data = fields.get(field) or {} + if not isinstance(field_data, dict): + field_data = {} + scores[field] = _safe_float(field_data.get("score"), default=1.0) + for issue in _normalize_issues(field_data.get("issues")): + issues.append(f"{field}:{issue}") + reason = str(field_data.get("reason") or "").strip() + if reason: + reasons.append(f"{field}: {reason}") + + for issue in _normalize_issues(data.get("overall_issues")): + issues.append(issue) + + return LLMFieldQuality( + title_score=scores["title"], + abstract_score=scores["abstract"], + keywords_score=scores["keywords"], + venue_score=scores["venue"], + author_score=scores["author"], + issues=issues, + reason=str(data.get("reason") or "; ".join(reasons))[:500], + ) + + +@dataclass +class EffectivenessGrade: + """Structured score for one search result.""" + + score: float = 0.0 + title_score: float = 0.0 + abstract_score: float = 0.0 + keywords_score: float = 0.0 + venue_score: float = 0.0 + author_score: float = 0.0 + issues: list[str] | None = None + llm_quality_reason: str = "" + llm_quality_error: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "score": round(self.score, 5), + "title_score": round(self.title_score, 5), + "abstract_score": round(self.abstract_score, 5), + "keywords_score": round(self.keywords_score, 5), + "venue_score": round(self.venue_score, 5), + "author_score": round(self.author_score, 5), + "issues": self.issues or [], + "llm_quality_reason": self.llm_quality_reason, + "llm_quality_error": self.llm_quality_error, + } + + +@dataclass +class EffectivenessSummary: + mean_score: float = 0.0 + median_score: float = 0.0 + mean_title_score: float = 0.0 + mean_abstract_score: float = 0.0 + mean_keywords_score: float = 0.0 + mean_venue_score: float = 0.0 + mean_author_score: float = 0.0 + graded_pairs: int = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "effectiveness_mean_score": round(self.mean_score, 5), + "effectiveness_median_score": round(self.median_score, 5), + "effectiveness_mean_title_score": round(self.mean_title_score, 5), + "effectiveness_mean_abstract_score": round(self.mean_abstract_score, 5), + "effectiveness_mean_keywords_score": round(self.mean_keywords_score, 5), + "effectiveness_mean_venue_score": round(self.mean_venue_score, 5), + "effectiveness_mean_author_score": round(self.mean_author_score, 5), + "effectiveness_graded_pairs": self.graded_pairs, + } + + +@Model.llm_register("LLMSearchResultEffectiveness") +class LLMSearchResultEffectiveness: + """Effectiveness scorer for title, abstract, keywords, and authors. + + Venue text is still scanned for corruption, but venue presence and quality + belong to the authority metric and do not affect the effectiveness score. + """ + + dynamic_config = EvaluatorLLMArgs() + default_threshold = 0.15 + + def __init__( + self, + *, + model: str | None = None, + api_key: str | None = None, + api_url: str | None = None, + max_tokens: int = 512, + temperature: float = 0.0, + timeout: float | None = None, + enable_llm_quality: bool = False, + ): + self.model = model or "gpt-4o" + self.api_key = api_key + self.api_url = api_url + self.max_tokens = max_tokens + self.temperature = temperature + self.timeout = timeout + self.enable_llm_quality = enable_llm_quality + self._client = None + + def _get_client(self): + if self._client is None: + from openai import OpenAI + + kwargs: dict[str, Any] = {} + if self.api_key: + kwargs["api_key"] = self.api_key + if self.api_url: + kwargs["base_url"] = self.api_url + self._client = OpenAI(**kwargs) + return self._client + + def _build_llm_quality_user_message( + self, + *, + title: str, + abstract: str, + keywords: list[str], + venue: str, + authors: list[str], + candidate_fields: set[str] | None = None, + ) -> str: + all_fields = { + "title": title, + "abstract": abstract, + "keywords": " | ".join(keywords), + "venue": venue, + "author": " | ".join(authors), + } + selected = candidate_fields or set(all_fields) + payload = { + field: _truncate_for_llm(value) + for field, value in all_fields.items() + if field in selected + } + return ( + "Evaluate only the supplied fields for readability and corruption. " + "Omitted fields should not be judged. Return compact JSON.\n\n" + f"{json.dumps(payload, ensure_ascii=False, indent=2)}" + ) + + def _judge_llm_field_quality( + self, + *, + title: str, + abstract: str, + keywords: list[str], + venue: str, + authors: list[str], + candidate_fields: set[str] | None = None, + ) -> LLMFieldQuality: + if not self.enable_llm_quality: + return LLMFieldQuality() + client = self._get_client() + last_result = LLMFieldQuality(error="LLM field quality judgment failed") + for attempt in range(3): + try: + completion = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": LLM_FIELD_QUALITY_SYSTEM_PROMPT}, + { + "role": "user", + "content": self._build_llm_quality_user_message( + title=title, + abstract=abstract, + keywords=keywords, + venue=venue, + authors=authors, + candidate_fields=candidate_fields, + ), + }, + ], + temperature=self.temperature, + max_tokens=self.max_tokens, + timeout=self.timeout, + ) + response_text = completion.choices[0].message.content or "" + last_result = _parse_llm_field_quality_response(response_text) + if not last_result.error: + return last_result + error: Exception | str = last_result.error + except Exception as exc: + error = exc + last_result = LLMFieldQuality(error=str(exc)) + + logger.warning( + "LLM field quality attempt %s/3 failed for title=%r: %s", + attempt + 1, + title, + error, + ) + if attempt < 2: + time.sleep(attempt + 1) + return last_result + + def grade( + self, + *, + title: str = "", + abstract: str = "", + keywords: list[str] | str | None = None, + venue: str = "", + authors: list[str] | str | None = None, + result: dict[str, Any] | None = None, + ) -> EffectivenessGrade: + if result is not None: + title = str(result.get("title") or result.get("display_name") or title or "") + abstract = str(result.get("abstract") or abstract or "") + keywords = extract_keywords(result) if keywords is None else keywords + venue = extract_venue(result) or venue + authors = extract_authors(result) if authors is None else authors + + keyword_items = ( + [item.strip() for item in re.split(r"[,;|]", keywords) if item.strip()] + if isinstance(keywords, str) + else [str(item).strip() for item in (keywords or []) if str(item).strip()] + ) + author_items = ( + [item.strip() for item in re.split(r"[;|]", authors) if item.strip()] + if isinstance(authors, str) + else [str(item).strip() for item in (authors or []) if str(item).strip()] + ) + + title_score = _presence_quality(title) + abstract_score = _presence_quality(abstract) + keywords_score = 1.0 if keyword_items else 0.0 + venue_score = _presence_quality(venue) + author_score = 1.0 if author_items else 0.0 + + issues: list[str] = [] + if not str(title or "").strip(): + issues.append("missing_title") + if not str(abstract or "").strip(): + issues.append("missing_abstract") + if not keyword_items: + issues.append("missing_keywords") + if not author_items: + issues.append("missing_author") + + field_values = { + "title": str(title or ""), + "abstract": str(abstract or ""), + "keywords": " | ".join(keyword_items), + "venue": str(venue or ""), + "author": " | ".join(author_items), + } + rule_candidate_issues = { + field: _rule_abnormal_char_issues(value) + for field, value in field_values.items() + if value + } + rule_candidate_issues = { + field: field_issues + for field, field_issues in rule_candidate_issues.items() + if field_issues + } + + llm_quality = LLMFieldQuality() + if rule_candidate_issues and self.enable_llm_quality: + llm_quality = self._judge_llm_field_quality( + title=str(title or ""), + abstract=str(abstract or ""), + keywords=keyword_items, + venue=str(venue or ""), + authors=author_items, + candidate_fields=set(rule_candidate_issues), + ) + + def apply_confirmed_field_issue(field: str, score: float) -> float: + field_rule_issues = rule_candidate_issues.get(field) or [] + if not field_rule_issues: + return score + + if not self.enable_llm_quality: + issues.extend(field_rule_issues) + return min(score, 0.1) + + if llm_quality.error: + return score + + field_llm_issues = [ + issue for issue in (llm_quality.issues or []) + if str(issue).startswith(f"{field}:") + ] + field_llm_issues = _filter_llm_field_issues( + field, + field_values.get(field, ""), + field_llm_issues, + ) + llm_field_score = llm_quality.field_score(field) + if llm_field_score < 1.0 or _has_confirmed_llm_issue(field_llm_issues): + issues.extend(field_rule_issues) + issues.extend(field_llm_issues) + return min(score, llm_field_score) + return score + + title_score = apply_confirmed_field_issue("title", title_score) + abstract_score = apply_confirmed_field_issue("abstract", abstract_score) + keywords_score = apply_confirmed_field_issue("keywords", keywords_score) + venue_score = apply_confirmed_field_issue("venue", venue_score) + author_score = apply_confirmed_field_issue("author", author_score) + + if rule_candidate_issues and self.enable_llm_quality and llm_quality.error: + issues.append("llm_quality_parse_error") + + score = ( + 0.30 * title_score + + 0.50 * abstract_score + + 0.10 * keywords_score + + 0.10 * author_score + ) + return EffectivenessGrade( + score=_clamp(score), + title_score=_clamp(title_score), + abstract_score=_clamp(abstract_score), + keywords_score=_clamp(keywords_score), + venue_score=_clamp(venue_score), + author_score=_clamp(author_score), + issues=issues, + llm_quality_reason=llm_quality.reason, + llm_quality_error=llm_quality.error, + ) + + @classmethod + def _config_value(cls, name: str, default: Any = None) -> Any: + return getattr(cls.dynamic_config, name, default) + + @classmethod + def _build_from_config(cls) -> "LLMSearchResultEffectiveness": + return cls( + model=cls.dynamic_config.model, + api_key=cls.dynamic_config.key, + api_url=cls.dynamic_config.api_url, + max_tokens=int(cls._config_value("max_tokens", 512) or 512), + temperature=float(cls._config_value("temperature", 0.0) or 0.0), + timeout=cls._config_value("timeout", None), + enable_llm_quality=bool(cls._config_value("enable_llm_quality", False)), + ) + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Executor entry point: evaluate one flattened search result row.""" + result = getattr(input_data, "search_result", None) + if not isinstance(result, dict): + result = input_data.to_dict() + + grader = cls._build_from_config() + grade = grader.grade(result=result) + threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold) + + labels = _issues_to_labels(grade.issues) + + if grade.score < threshold and "Effectiveness.Error_Effectiveness_Low" not in labels: + labels.append("Effectiveness.Error_Effectiveness_Low") + + status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + + return EvalDetail( + metric=cls.__name__, + status=status, + score=round(grade.score, 5), + label=labels, + reason=[grade.to_dict()], + ) + + +def aggregate_grades(grades: list[EffectivenessGrade]) -> EffectivenessSummary: + if not grades: + return EffectivenessSummary() + return EffectivenessSummary( + mean_score=statistics.mean(g.score for g in grades), + median_score=statistics.median(g.score for g in grades), + mean_title_score=statistics.mean(g.title_score for g in grades), + mean_abstract_score=statistics.mean(g.abstract_score for g in grades), + mean_keywords_score=statistics.mean(g.keywords_score for g in grades), + mean_venue_score=statistics.mean(g.venue_score for g in grades), + mean_author_score=statistics.mean(g.author_score for g in grades), + graded_pairs=len(grades), + ) diff --git a/dingo/model/llm/llm_search_result_relevance.py b/dingo/model/llm/llm_search_result_relevance.py index d3f1a62b..febea86f 100644 --- a/dingo/model/llm/llm_search_result_relevance.py +++ b/dingo/model/llm/llm_search_result_relevance.py @@ -17,25 +17,40 @@ from __future__ import annotations import json import logging +import re import statistics +import time from dataclasses import dataclass from typing import Any +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model import Model + logger = logging.getLogger(__name__) -STANDARD_SYSTEM_PROMPT = """\ -You are a helpful assistant that grades the relevance of search results for given queries. -Your task is to assign a relevance score between 0.0 and 1.0 to each result, based on -how good a result is for the query. +HTML_TAG_PATTERN = r"<[^>]+>" +MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}|�{2,}" +INVISIBLE_CHAR_PATTERN = r"[\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]" +DOI_PATTERN = re.compile(r"(?i)(?:https?://(?:dx\.)?doi\.org/)?(10\.\d{4,9}/[^\s]+)") -For each search result, carefully read the query and the result. Assign a value for -each criterion as follows: -- Provide a brief explanation of your reasoning. +STANDARD_SYSTEM_PROMPT = """\ +Grade how useful a search result is for a query and assign a relevance score +from 0.0 to 1.0. +Read the query, title, and available content, then return these fields: +- Brief reasoning in 20 words or fewer. - Assign a query_relevance score between 0.0 and 1.0. - Assign a result_quality score between 0.0 and 1.0. -- Indicate if there are any content_issues (true/false). +- Set content_issues to true or false. - Assign a confidence score between 0.0 and 1.0. -- Assign an overall score between 0.0 and 1.0.""" +- Assign an overall score between 0.0 and 1.0. + +Set content_issues=true only when mojibake, raw HTML/XML, parser residue, invisible/control +characters, or similar corruption makes visible content materially unreadable. Missing or short +abstracts and truncated previews are not issues when the visible title/snippet remains readable. + +Return one valid JSON object only. Do not use double quotes inside reasoning.""" DETAILED_SYSTEM_PROMPT = """\ You are a helpful assistant that grades the relevance of search results for given queries. @@ -73,9 +88,11 @@ 2. result_quality: The authority, accuracy, and trustworthiness of the result. High-quality \ results come from reputable sources, are well-written, and are not spammy or misleading. -3. content_issues: A boolean indicating whether there are problems with the content, such as \ -truncation, missing information, or improper parsing. If the result is incomplete or garbled, \ -set this to true. +3. content_issues: A boolean indicating severe content corruption only. Set this to true \ +when the visible title/content has garbled or mojibake text, raw HTML/XML or parser residue \ +that materially hurts readability, invisible/control characters, or unreadable text. Do NOT \ +set this to true merely because the abstract is missing, the snippet is short, or the preview \ +is truncated, if the visible title/snippet is still readable enough to judge relevance. 4. confidence: How certain you are about your grading. If the result snippet is clear and \ directly answers the query, confidence should be high. If you need external information to \ @@ -86,15 +103,18 @@ For each search result, carefully read the query and the result. Assign a value for \ each criterion as follows: -- Provide a brief explanation of your reasoning. +- Provide a brief explanation of your reasoning in 20 words or fewer. - Assign a query_relevance score between 0.0 and 1.0. - Assign a result_quality score between 0.0 and 1.0. -- Indicate if there are any content_issues (true/false). +- Indicate if there are severe content_issues (true/false). - Assign a confidence score between 0.0 and 1.0. - Assign an overall score between 0.0 and 1.0. Be consistent and use decimal points for fine-grained differentiation. If you are unsure \ -due to missing or unclear information, lower your confidence and make a best guess as to the score.""" +due to missing or unclear information, lower your confidence and make a best guess as to the score. + +Return only one valid JSON object. Keep reasoning short and do not use double quotes inside \ +the reasoning string. If you need quotation marks in reasoning, use single quotes.""" @dataclass @@ -179,11 +199,13 @@ def _build_user_message( '"result_quality": 0.0-1.0, "content_issues": true/false, ' '"confidence": 0.0-1.0, "score": 0.0-1.0}' ) + parts.append( + "Return JSON only. Keep reasoning under 20 words and do not use double quotes inside reasoning." + ) return "\n".join(parts) -def _parse_grade_response(response_text: str) -> RelevanceGrade: - """Parse LLM JSON response into a RelevanceGrade.""" +def _strip_json_fence(response_text: str) -> str: text = response_text.strip() if text.startswith("```json"): text = text[7:] @@ -191,28 +213,209 @@ def _parse_grade_response(response_text: str) -> RelevanceGrade: text = text[3:] if text.endswith("```"): text = text[:-3] - text = text.strip() + return text.strip() - try: - data = json.loads(text) - except json.JSONDecodeError: - return RelevanceGrade(error=f"JSON parse failed: {text[:200]}") +def _extract_json_object(text: str) -> str | None: + start = text.find("{") + end = text.rfind("}") + if start == -1 or end == -1 or end <= start: + return None + return text[start:end + 1].strip() + + +def _repair_unescaped_quotes_in_reasoning(text: str) -> str: + """Escape stray double quotes inside the reasoning JSON string. + + Some models emit otherwise valid JSON such as: + {"reasoning": "The query "PBPK" matches", "score": 0.9, ...} + The inner quotes break json.loads. This repair scopes the change to the + reasoning value and leaves the following JSON keys untouched. + """ + start_match = re.search(r'("reasoning"\s*:\s*")', text) + if not start_match: + return text + + value_start = start_match.end() + next_key = re.search( + r'"\s*(?:,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"\s*:|})', + text[value_start:], + flags=re.DOTALL, + ) + if not next_key: + return text + + value_end = value_start + next_key.start() + value = text[value_start:value_end] + repaired_value = re.sub(r'(? float: try: - if not isinstance(data, dict): - return RelevanceGrade(error=f"JSON is not a dictionary: {text[:200]}") - return RelevanceGrade( - score=float(data.get("score", 0.0)), - query_relevance=float(data.get("query_relevance", 0.0)), - result_quality=float(data.get("result_quality", 0.0)), - content_issues=bool(data.get("content_issues", False)), - confidence=float(data.get("confidence", 0.0)), - reasoning=str(data.get("reasoning", "")), + return float(value) + except (TypeError, ValueError): + return 0.0 + + +def _clamp_score(value: Any) -> float: + return max(0.0, min(1.0, _coerce_float(value))) + + +def _grade_from_dict(data: dict[str, Any]) -> RelevanceGrade: + return RelevanceGrade( + score=_clamp_score(data.get("score", 0.0)), + query_relevance=_clamp_score(data.get("query_relevance", 0.0)), + result_quality=_clamp_score(data.get("result_quality", 0.0)), + content_issues=bool(data.get("content_issues", False)), + confidence=_clamp_score(data.get("confidence", 0.0)), + reasoning=str(data.get("reasoning", "")), + ) + + +def _parse_grade_fields_lenient(text: str) -> RelevanceGrade | None: + number_fields = {} + for field in ("query_relevance", "result_quality", "confidence", "score"): + match = re.search( + rf'"{field}"\s*:\s*([-+]?(?:\d+(?:\.\d*)?|\.\d+))', + text, + flags=re.IGNORECASE, ) - except (ValueError, TypeError) as e: - return RelevanceGrade(error=f"Failed to parse grade response: {e}. Text: {text[:200]}") + if match: + number_fields[field] = _clamp_score(match.group(1)) + + if "score" not in number_fields and "query_relevance" not in number_fields: + return None + + issue_match = re.search(r'"content_issues"\s*:\s*(true|false)', text, flags=re.IGNORECASE) + reasoning = "" + reasoning_match = re.search( + r'"reasoning"\s*:\s*"(.*?)"\s*(?:,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"|})', + text, + flags=re.DOTALL, + ) + if reasoning_match: + reasoning = " ".join(reasoning_match.group(1).split()) + + return RelevanceGrade( + score=number_fields.get("score", number_fields.get("query_relevance", 0.0)), + query_relevance=number_fields.get("query_relevance", 0.0), + result_quality=number_fields.get("result_quality", 0.0), + content_issues=issue_match.group(1).lower() == "true" if issue_match else False, + confidence=number_fields.get("confidence", 0.0), + reasoning=reasoning, + ) + + +def _parse_grade_response(response_text: str) -> RelevanceGrade: + """Parse LLM JSON response into a RelevanceGrade.""" + text = _strip_json_fence(response_text) + candidates = [text] + extracted = _extract_json_object(text) + if extracted and extracted != text: + candidates.append(extracted) + + repaired_candidates = [] + for candidate in candidates: + repaired = _repair_unescaped_quotes_in_reasoning(candidate) + if repaired != candidate: + repaired_candidates.append(repaired) + candidates.extend(repaired_candidates) + + for candidate in candidates: + try: + data = json.loads(candidate) + except json.JSONDecodeError: + continue + + try: + if not isinstance(data, dict): + return RelevanceGrade(error=f"JSON is not a dictionary: {candidate[:200]}") + return _grade_from_dict(data) + except (ValueError, TypeError) as e: + return RelevanceGrade(error=f"Failed to parse grade response: {e}. Text: {candidate[:200]}") + + lenient = _parse_grade_fields_lenient(extracted or text) + if lenient: + return lenient + return RelevanceGrade(error=f"JSON parse failed: {text[:200]}") + + +def _content_issue_evidence(title: str, abstract: str) -> list[str]: + text = "\n".join([str(title or ""), str(abstract or "")]) + issues: list[str] = [] + if re.search(MOJIBAKE_EVIDENCE_PATTERN, text): + issues.append("mojibake_or_garbled_text") + if re.search(INVISIBLE_CHAR_PATTERN, text): + issues.append("invisible_or_control_char") + html_matches = re.findall(HTML_TAG_PATTERN, text) + if html_matches: + issues.append("html_or_xml_tag_residue") + if re.search(r"(/docserver/|<\?xml| bool: + return bool(_content_issue_evidence(title, abstract)) + + +def _normalize_doi(value: Any) -> str: + """Extract and normalize a DOI from a query or result field.""" + text = str(value or "").strip() + match = DOI_PATTERN.search(text) + if not match: + return "" + return match.group(1).rstrip(".,;:)]}").lower() + + +def is_doi_query(query: str) -> bool: + return bool(_normalize_doi(query)) + + +def _extract_result_dois(result: dict[str, Any]) -> list[str]: + candidates: list[Any] = [result.get("doi"), result.get("unique_id")] + locations = result.get("locations") + if isinstance(locations, list): + for location in locations: + if isinstance(location, dict): + candidates.extend([location.get("doi"), location.get("url"), location.get("landing_page_url")]) + + dois: list[str] = [] + for candidate in candidates: + values = candidate if isinstance(candidate, list) else [candidate] + for value in values: + doi = _normalize_doi(value) + if doi and doi not in dois: + dois.append(doi) + return dois + + +def _grade_doi_result(query: str, result: dict[str, Any]) -> RelevanceGrade | None: + """Use deterministic identifier matching when the query is a DOI.""" + query_doi = _normalize_doi(query) + if not query_doi: + return None + + result_dois = _extract_result_dois(result) + exact_match = query_doi in result_dois + score = 1.0 if exact_match else 0.0 + result_text = ", ".join(result_dois) if result_dois else "missing" + return RelevanceGrade( + score=score, + query_relevance=score, + result_quality=1.0 if exact_match else 0.0, + content_issues=False, + confidence=1.0, + reasoning=( + f"Exact DOI match: {query_doi}." + if exact_match + else f"DOI mismatch: expected {query_doi}; result {result_text}." + ), + ) +@Model.llm_register("LLMSearchResultRelevance") class LLMSearchResultRelevance: """Exa-style pointwise search result relevance grader. @@ -220,6 +423,9 @@ class LLMSearchResultRelevance: ``BaseOpenAI`` evaluator hierarchy. """ + dynamic_config = EvaluatorLLMArgs() + default_threshold = 0.15 + def __init__( self, *, @@ -228,12 +434,18 @@ def __init__( api_url: str | None = None, prompt_mode: str = "standard", expected_criteria: str | None = None, + max_tokens: int = 1024, + temperature: float = 0.0, + timeout: float | None = None, ): self.model = model or "gpt-4o" self.api_key = api_key self.api_url = api_url self.prompt_mode = prompt_mode self.expected_criteria = expected_criteria + self.max_tokens = max_tokens + self.temperature = temperature + self.timeout = timeout self._client = None def _get_client(self): @@ -261,22 +473,111 @@ def grade( expected_criteria=expected_criteria or self.expected_criteria, ) - try: - client = self._get_client() - completion = client.chat.completions.create( - model=self.model, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_message}, - ], - temperature=0.0, - max_tokens=512, + client = self._get_client() + last_grade = RelevanceGrade(error="LLM grading failed") + for attempt in range(3): + try: + completion = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_message}, + ], + temperature=self.temperature, + max_tokens=self.max_tokens, + timeout=self.timeout, + ) + response_text = completion.choices[0].message.content or "" + last_grade = _parse_grade_response(response_text) + if not last_grade.error: + return last_grade + error: Exception | str = last_grade.error + except Exception as exc: + error = exc + last_grade = RelevanceGrade(error=str(exc)) + + logger.warning( + "LLM grading attempt %s/3 failed for query=%r title=%r: %s", + attempt + 1, + query, + title, + error, ) - response_text = completion.choices[0].message.content or "" - return _parse_grade_response(response_text) - except Exception as e: - logger.warning("LLM grading failed for query=%r title=%r: %s", query, title, e) - return RelevanceGrade(error=str(e)) + if attempt < 2: + time.sleep(attempt + 1) + return last_grade + + @classmethod + def _config_value(cls, name: str, default: Any = None) -> Any: + return getattr(cls.dynamic_config, name, default) + + @classmethod + def _build_from_config(cls) -> "LLMSearchResultRelevance": + return cls( + model=cls.dynamic_config.model, + api_key=cls.dynamic_config.key, + api_url=cls.dynamic_config.api_url, + prompt_mode=str(cls._config_value("prompt_mode", "detailed") or "detailed"), + expected_criteria=cls._config_value("expected_criteria", None), + max_tokens=int(cls._config_value("max_tokens", 1024) or 1024), + temperature=float(cls._config_value("temperature", 0.0) or 0.0), + timeout=cls._config_value("timeout", None), + ) + + @staticmethod + def _extract_title(result: dict[str, Any]) -> str: + return str(result.get("title") or result.get("display_name") or "") + + @staticmethod + def _extract_abstract(result: dict[str, Any]) -> str: + return str(result.get("abstract") or result.get("summary") or result.get("content") or "") + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Executor entry point: evaluate one flattened query-result pair.""" + result = getattr(input_data, "search_result", None) + if not isinstance(result, dict): + result = input_data.to_dict() + query = str(getattr(input_data, "query", "") or result.get("_eval_query") or result.get("query") or "") + + title = cls._extract_title(result) + abstract = cls._extract_abstract(result) + grade = _grade_doi_result(query, result) + if grade is None: + grader = cls._build_from_config() + grade = grader.grade( + query=query, + title=title, + abstract=abstract, + ) + threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold) + content_issue_evidence = _content_issue_evidence(title, abstract) if grade.content_issues else [] + effective_content_issues = bool(content_issue_evidence) + + labels: list[str] = [] + if grade.error: + labels.append("Relevance.Error_Parse") + if grade.score < threshold: + labels.append("Relevance.Error_Relevance_Low") + if effective_content_issues: + labels.append("Relevance.Error_Content_Issues") + + status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + + reason = grade.to_dict() + reason["raw_content_issues"] = grade.content_issues + reason["content_issues"] = effective_content_issues + reason["content_issue_evidence"] = content_issue_evidence + + return EvalDetail( + metric=cls.__name__, + status=status, + score=round(grade.score, 5), + label=labels, + reason=[reason], + ) def aggregate_grades( diff --git a/docs/search_result_authority_executor.md b/docs/search_result_authority_executor.md new file mode 100644 index 00000000..3b171df4 --- /dev/null +++ b/docs/search_result_authority_executor.md @@ -0,0 +1,187 @@ +# Search Result Authority Executor Usage + +This document describes the executor-based authority evaluation for meta search results. + +## What Changed + +`examples/retrieval/sdk_eval_authority.py` evaluates each retrieved result with `LLMSearchResultAuthority` through Dingo `LocalExecutor`. + +The standalone authority script evaluates each retrieved result as one test object. The combined script `sdk_eval_search_result.py` still evaluates at query level and applies query-level bad/good thresholds. + +`LLMSearchResultAuthority` is rule-based despite being registered as an LLM evaluator. It does not call an external model. The score is based on citation impact, venue/source signals, and DOI availability. + +## Flow + +1. Read query-level meta search JSONL. +2. Flatten each query's top-k results into result-level JSONL rows. +3. Build `InputArgs`. +4. Run `Executor.exec_map["local"]`. +5. `LLMSearchResultAuthority` returns standard `EvalDetail(metric/status/score/label/reason)`. +6. `LocalExecutor` writes timestamped output, `summary.json`, `search_result/QUALITY_GOOD.jsonl`, and `search_result/Authority/Error_*.jsonl`. + +Each flattened input row contains: + +| Field | Meaning | +|---|---| +| `query` | Original query text | +| `query_index` | Query order in source JSONL | +| `rank` | Result rank under the query | +| `title` | Result title or display name | +| `search_result` | Full result payload passed to evaluator | + +## Test Data + +The repository includes `test/data/test_search_result.jsonl` with three queries and nine results. It covers normal papers (`BiMLP`), search-highlight HTML (`海带`), and sparse ebook metadata (`pam`). Authority evaluation is rule-based, so this smoke test does not require an LLM API: + +```powershell +python examples/retrieval/sdk_eval_authority.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_authority_smoke ` + --top-k 3 ` + --threshold 0.15 ` + --save-good +``` + +## Commands + +Run authority evaluation: + +```powershell +python examples/retrieval/sdk_eval_authority.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_authority_97q_executor ` + --top-k 10 ` + --threshold 0.15 ` + --save-good +``` + +Smoke test with fewer queries: + +```powershell +python examples/retrieval/sdk_eval_authority.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_authority_smoke ` + --top-k 10 ` + --max-queries 5 ` + --threshold 0.15 ` + --save-good +``` + +Useful parameters: + +| Parameter | Default | Meaning | +|---|---:|---| +| `--top-k` | `10` | Number of results evaluated per query | +| `--max-queries` | `None` | Limit query count for smoke tests | +| `--threshold` | `0.15` | Result-level bad threshold | +| `--max-workers` | `4` | LocalExecutor worker count | +| `--batch-size` | `10` | LocalExecutor batch size | +| `--save-good` | off | Save passing samples | +| `--raw-output` | off | Merge raw data and Dingo result in output JSONL rows | + +## Output + +The executor creates a timestamped child directory under `--output-dir`, for example: + +```text +outputs/search_result_authority_97q_executor/20260709_172501_0f73c631/ +``` + +Main files: + +| Path | Meaning | +|---|---| +| `summary.json` | Executor summary | +| `search_result/QUALITY_GOOD.jsonl` | Passing result-level samples, only with `--save-good` | +| `search_result/Authority/Error_*.jsonl` | Bad result-level samples grouped by error type | + +`summary.json` uses result-level statistics: + +- `total`: number of evaluated retrieved results. +- `num_good`: result count with no error labels. +- `num_bad`: result count with at least one error label. +- `score`: `num_good / total * 100`. +- `metrics_score.search_result.stats.LLMSearchResultAuthority`: result-level authority score distribution. +- `type_ratio.search_result`: label ratios. One result can have multiple error labels, so error ratios can sum to more than the bad ratio. + +Error labels: + +| Label path | Trigger | +|---|---| +| `search_result/Authority/Error_Authority_Low.jsonl` | Final authority score below threshold | +| `search_result/Authority/Error_Citation_Miss.jsonl` | Authority is low and citation score is zero | +| `search_result/Authority/Error_Venue_Low_Signal.jsonl` | Authority is low and venue/source has only low signal | +| `search_result/Authority/Error_DOI_Miss.jsonl` | Authority is low and no DOI signal is present | + +The sub-labels are emitted only when the final authority score is below the threshold. For example, a result without DOI can still be `QUALITY_GOOD` if citation and venue signals are strong enough. + +## Scoring + +The metric score is: + +```text +authority = + 0.45 * citation_score ++ 0.20 * influential_citation_score ++ 0.25 * venue_score ++ 0.10 * doi_score +``` + +Citation scores use log normalization and are clamped to `[0, 1]`: + +```text +citation_score = log1p(citation_count) / log1p(500) +influential_citation_score = log1p(influential_citation_count) / log1p(50) +``` + +Venue is read from the first available field: + +```text +publication_venue_name_unified +publication_venue_name +venue +source +``` + +Venue scoring: + +| Condition | `venue_score` | Reason | +|---|---:|---| +| Known repository or preprint source | `0.45` | `repository_or_preprint` | +| Explicit academic book series or ebook platform | `0.55` | `academic_book_series` | +| Venue matches a prestigious journal/conference family | `0.85` | `prestigious_venue_family` | +| Venue or publisher matches a recognized scholarly organization | `0.75` | `recognized_scholarly_publisher_or_venue` | +| Journal/conference type or valid ISSN is present | `0.65` | `structured_journal_or_conference` | +| A venue name is present without stronger structured signals | `0.40` | `named_venue` | +| Unknown or low-signal source | `0.25` | `unknown_or_low_signal_venue` | + +Explicit source types run first. Repository detection prevents names such as `Open Science Framework` from being promoted merely because they contain a prestigious-looking word, while book series and ebook platforms remain at the book-source tier even when their publisher is recognized. + +Prestigious venue families include: + +```text +Nature and Nature subject journals, Nature Communications, +npj journals, Communications journals, Scientific Reports/Data, +the official Science journal family, selected Cell Press flagships, +NEJM, Lancet, JAMA, The BMJ, PNAS, JACS, PRL, and major AI conferences +``` + +The matcher uses anchored family patterns instead of unrestricted substrings. For example, `Science Translational Medicine` matches the Science family, while `Chemical Engineering Science` does not. HTML highlight tags are removed before matching. + +Recognized publisher metadata includes established scholarly publishers and societies such as Springer Nature, Elsevier, Wiley, Oxford University Press, Cambridge University Press, IEEE, ACM, ACS, RSC, IOP, BMJ, PLOS, the Royal Society, De Gruyter, CRC Press, and World Scientific. Publisher recognition is deliberately scored below an explicitly prestigious venue because publisher reputation alone does not make every title a flagship journal. + +The ISSN and venue-type fallback is the main protection against an incomplete whitelist: a journal does not need to appear in a hard-coded title list to receive a structured scholarly venue score. + +DOI scoring: + +```text +doi_score = 1.0 +``` + +when the result has a `doi` field or `locations` contains `doi.org`; otherwise: + +```text +doi_score = 0.0 +``` + +Authority does not judge query relevance or content completeness. A low authority score means the result lacks academic trust signals such as citations, venue, or DOI. It does not necessarily mean the result is irrelevant or unusable. diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md new file mode 100644 index 00000000..432e0d4f --- /dev/null +++ b/docs/search_result_effectiveness_executor.md @@ -0,0 +1,152 @@ +# Search Result Effectiveness Executor Usage + +This document describes the executor-based effectiveness evaluation for meta search results. + +## What Changed + +`examples/retrieval/sdk_eval_effectiveness.py` reuses Dingo `LocalExecutor`. It no longer hand-builds chunk-style `summary.json` or bad/good folders. + +The standalone effectiveness script evaluates each retrieved result as one test object. The combined script `sdk_eval_search_result.py` still evaluates at query level and applies query-level bad/good thresholds. + +## Flow + +1. Read query-level meta search JSONL. +2. Flatten each query's top-k results into result-level JSONL rows. +3. Build `InputArgs`. +4. Run `Executor.exec_map["local"]`. +5. `LLMSearchResultEffectiveness` returns standard `EvalDetail(metric/status/score/label/reason)`. +6. `LocalExecutor` writes timestamped output, `summary.json`, `search_result/QUALITY_GOOD.jsonl`, and `search_result/Effectiveness/Error_*.jsonl`. + +Each flattened input row contains: + +| Field | Meaning | +|---|---| +| `query` | Original query text | +| `query_index` | Query order in source JSONL | +| `rank` | Result rank under the query | +| `title` | Result title or display name | +| `search_result` | Full result payload passed to evaluator | + +## Test Data + +The repository includes `test/data/test_search_result.jsonl` with three queries and nine results. It covers normal papers (`BiMLP`), search-highlight HTML (`海带`), and sparse ebook metadata (`pam`). + +Smoke test with LLM second judgment enabled: + +```powershell +python examples/retrieval/sdk_eval_effectiveness.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_effectiveness_smoke ` + --top-k 3 ` + --llm-max-tokens 1024 ` + --threshold 0.15 ` + --save-good +``` + +## Commands + +Fast rule-only run: + +```powershell +python examples/retrieval/sdk_eval_effectiveness.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_effectiveness_97q_executor ` + --top-k 10 ` + --threshold 0.15 ` + --disable-llm-quality ` + --save-good +``` + +Run with LLM second judgment for abnormal-character candidates: + +For full runs, use a low-latency Flash model such as `deepseek-v4-flash`. The LLM is only used to review rule-selected suspicious fields, but a slow Pro model can still substantially increase runtime. Reserve Pro models for small-sample diagnosis, and use temperature `0` for reproducible comparisons. + +```powershell +$env:OPENAI_API_KEY="..." +$env:OPENAI_BASE_URL="http://35.220.164.252:3888/v1/" +$env:OPENAI_MODEL="deepseek-v4-flash" +$env:OPENAI_TEMPERATURE="0" + +python examples/retrieval/sdk_eval_effectiveness.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_effectiveness_97q_executor_llm ` + --top-k 10 ` + --threshold 0.15 ` + --llm-max-tokens 512 ` + --llm-workers 4 ` + --save-good +``` + +Useful parameters: + +| Parameter | Default | Meaning | +|---|---:|---| +| `--top-k` | `10` | Number of results evaluated per query | +| `--max-queries` | `None` | Limit query count for smoke tests | +| `--threshold` | `0.15` | Result-level bad threshold | +| `--save-good` | off | Save passing samples | +| `--disable-llm-quality` | off | Skip LLM second judgment and use deterministic rules only | +| `--llm-max-tokens` | `512` | Max tokens for LLM second judgment | +| `--llm-workers` | `4` | LocalExecutor worker count | +| `--batch-size` | `10` | LocalExecutor batch size | +| `--raw-output` | off | Merge raw data and Dingo result in output JSONL rows | + +## Output + +The executor creates a timestamped child directory under `--output-dir`, for example: + +```text +outputs/search_result_effectiveness_97q_executor/20260709_172501_0f73c631/ +``` + +Main files: + +| Path | Meaning | +|---|---| +| `summary.json` | Executor summary | +| `search_result/QUALITY_GOOD.jsonl` | Passing result-level samples, only with `--save-good` | +| `search_result/Effectiveness/Error_*.jsonl` | Bad result-level samples grouped by error type | + +`summary.json` uses result-level statistics: + +- `total`: number of evaluated retrieved results. +- `num_good`: result count with no error labels. +- `num_bad`: result count with at least one error label. +- `score`: `num_good / total * 100`. +- `metrics_score.search_result.stats.LLMSearchResultEffectiveness`: result-level effectiveness score distribution. +- `type_ratio.search_result`: label ratios. One result can have multiple error labels, so error ratios can sum to more than the bad ratio. + +Error labels: + +| Label path | Trigger | +|---|---| +| `search_result/Effectiveness/Error_Title_Miss.jsonl` | Missing title | +| `search_result/Effectiveness/Error_Abstract_Miss.jsonl` | Missing abstract | +| `search_result/Effectiveness/Error_Keywords_Miss.jsonl` | Missing keywords | +| `search_result/Effectiveness/Error_Author_Miss.jsonl` | Missing author metadata | +| `search_result/Effectiveness/Error_HTML_Tag.jsonl` | LLM-confirmed HTML tag pollution | +| `search_result/Effectiveness/Error_Mojibake.jsonl` | LLM-confirmed mojibake | +| `search_result/Effectiveness/Error_Invisible_Char.jsonl` | LLM-confirmed invisible characters | +| `search_result/Effectiveness/Error_Unreadable_Text.jsonl` | LLM-confirmed unreadable text | +| `search_result/Effectiveness/Error_Special_Char_Noise.jsonl` | LLM-confirmed special-character noise | +| `search_result/Effectiveness/Error_Effectiveness_Low.jsonl` | Final effectiveness score below threshold | + +`RuleSpecialCharacter` and `RuleInvisibleChar` are treated as candidate triggers. If LLM confirms a concrete issue, for example `title:html_tag`, the final label keeps only the concrete business label such as `Effectiveness.Error_HTML_Tag` and suppresses the intermediate rule label. The original rule issue is still retained in `EvalDetail.reason` for traceability. + +## Scoring + +The metric score is: + +```text +Effectiveness = + title_score * 0.30 ++ abstract_score * 0.50 ++ keywords_score * 0.10 ++ author_score * 0.10 +``` + +Each non-empty title, abstract, keywords list, or author list receives full presence credit before corruption checks. Character length, token count, keyword count, and author count do not affect the base score. A confirmed HTML, mojibake, invisible-character, unreadable-text, or special-character issue can still reduce the affected field score. + +`author_score` checks whether at least one non-empty author value is present. A single author receives full presence credit; the metric does not reward a larger author count or penalize a short name. + +Title, abstract, keywords, and author use presence checks and contribute to the score. Venue presence and quality are evaluated by the authority metric, so a missing venue does not reduce effectiveness or emit `Error_Venue_Miss`. All five text fields, including a venue when present, still use abnormal-character handling. HTML tags and the Unicode replacement character (`�`) always trigger LLM review, even when they occupy less than the general abnormal-character threshold. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md new file mode 100644 index 00000000..86cbf054 --- /dev/null +++ b/docs/search_result_quality_metrics.md @@ -0,0 +1,700 @@ +# Search Result Quality 三指标评测说明 + +本文档说明 meta search 检索结果的三类评测指标：相关性、内容有效性、权威性，以及对应的单项评测脚本和综合评测脚本。该方案面向无人工 GT 的检索结果质量检查，输入为 query 及其 top-k 检索结果，输出 query 级和 result 级分数，并按阈值生成 Dingo 风格的 good/bad 分类目录。 + +## 1. 适用场景 + +该评测用于回答三个业务问题： + +| 指标 | 业务问题 | 评测方式 | +|---|---|---| +| 相关性 `relevance` | 检索结果是否回答了用户 query 的真实检索意图 | LLM 逐条判断 query-result 匹配程度 | +| 内容有效性 `effectiveness` | 结果记录本身是否完整、可读、可用于判断论文价值 | 规则检查字段缺失，RuleSpecialCharacter/RuleInvisibleChar 初筛异常候选，LLM 二次确认；不使用字段长度打分 | +| 权威性 `authority` | 结果是否具备学术可信度和来源影响力信号 | 规则检查 citation、influential citation、venue、DOI | + +三个指标关注点不同： + +- 相关性判断“是不是用户要找的内容”。 +- 内容有效性判断“这条结果记录是否有足够信息可读可用”。 +- 权威性判断“这条结果是否有论文影响力、来源、DOI 等可信信号”。 + +例如，用户搜索 `Wallace Chafe`，rank1 返回标题也是 `Wallace Chafe`，相关性可能较好；但如果该结果没有 abstract、keywords、author，则内容有效性会较低。Publication venue 的缺失由权威性指标负责。 + +## 2. 输入格式 + +输入文件为 JSONL，每行一个 query 及其检索结果。 + +```json +{"query": "PBPK Review", "results": [{"title": "...", "abstract": "..."}]} +``` + +脚本支持的 query 字段名： + +- `query` +- `query_text` +- `q` + +脚本支持的结果列表字段名： + +- `results` +- `top_results` +- `top_api_results` +- `search_results` + +常用输入路径示例： + +```bash +outputs/meta_search_97_query_results.jsonl +``` + +## 3. 输出文件 + +综合评测脚本 `sdk_eval_search_result.py` 会在 `--output-dir` 下生成一个时间戳子目录，核心结果都放在该子目录中，例如： + +```text +outputs/search_result_eval_97q/20260710_162652_1ac3f3be/ +``` + +默认核心文件如下： + +| 文件 | 粒度 | 说明 | +|---|---|---| +| `summary.json` | 全局 | 指标均值、中位数、最小值、最大值、bad/good 数量、阈值、LLM 配置等 | +| `query_scores.csv` | query 级 | 每个 query 的 rank-discount 汇总分、label、eval_status | +| `result_scores.csv` | result 级 | 每个 query 的每条 top-k 结果分数和诊断信息 | +| `all_results.jsonl` | result 级原始明细 | executor 输出的逐条评测结果，保留三个指标的完整 `eval_details` | +| `bad/` | query 级分类 | 低于阈值或运行异常的 query 记录 | +| `good/` | query 级分类 | 使用 `--save-good` 时保存通过的 query 记录 | + +`detailed_results.json` 默认不生成；需要完整嵌套诊断时加 `--save-detailed`。 + +单独运行 `sdk_eval_effectiveness.py`、`sdk_eval_relevancy.py`、`sdk_eval_authority.py` 时，会保留 Dingo executor 的单指标 result 级分类目录，例如 `search_result/Effectiveness/Error_*.jsonl`。 + +分类 label 示例： + +```text +QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW +QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR +QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW +QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW +QUALITY_BAD.SEARCH_RESULT_OVERALL_LOW +QUALITY_GOOD.SEARCH_RESULT_RELEVANCE_PASS +QUALITY_GOOD.SEARCH_RESULT_EFFECTIVENESS_PASS +QUALITY_GOOD.SEARCH_RESULT_AUTHORITY_PASS +QUALITY_GOOD.SEARCH_RESULT_OVERALL_PASS +``` + +单独运行 `sdk_eval_effectiveness.py` 时，`summary.json` 采用类似 `sdk_chunk_eval.py` 的 result 级结构：每个 query 的每条 top-k 检索文献都是一个测试对象，`total`、`num_good`、`num_bad` 和 `score` 都按 result 级计算。`type_ratio.search_result` 中会统计 `Effectiveness.Error_*` 和 `QUALITY_GOOD` 的比例，`metrics_score.search_result` 中会统计 `LLMSearchResultEffectiveness` 的 result 级分数分布。 + +内容有效性还会额外输出 result 级错误类型文件，例如： + +```text +Effectiveness/Error_Title_Miss.jsonl +Effectiveness/Error_Abstract_Miss.jsonl +Effectiveness/Error_Keywords_Miss.jsonl +Effectiveness/Error_Author_Miss.jsonl +Effectiveness/Error_HTML_Tag.jsonl +Effectiveness/Error_Mojibake.jsonl +Effectiveness/Error_Invisible_Char.jsonl +Effectiveness/Error_Unreadable_Text.jsonl +Effectiveness/Error_Special_Char_Noise.jsonl +Effectiveness/Error_LLM_Quality_Parse.jsonl +QUALITY_GOOD.jsonl +``` + +只有实际出现的错误类型会生成对应文件。 + +## 4. Query 级汇总逻辑 + +三个指标都先对 top-k 中每条 result 打分，然后用 rank-discounted mean 汇总到 query 级。 + +第 `rank` 条结果的权重为： + +```text +weight(rank) = 1 / log2(rank + 1) +``` + +query 级分数为： + +```text +query_score = sum(result_score_i * weight_i) / sum(weight_i) +``` + +业务含义： + +- rank1 的影响最大。 +- rank 越靠后，对 query 总分影响越小。 +- 适合评估搜索排序质量，因为用户更关注前几条结果。 + +## 5. 相关性 Relevance + +### 5.1 业务逻辑 + +相关性判断每条检索结果与 query 是否匹配，重点看： + +- result 是否直接回答或覆盖 query 意图。 +- 标题和摘要是否围绕 query 主题。 +- 对短词、人名、论文题名、中文长 query 等非 DOI query，LLM 根据语义进行判断。 +- DOI query 使用结构化精确匹配：规范化 query DOI 与结果 `doi`、`unique_id` 或 location DOI，完全一致才算命中。 +- DOI 前缀相似、同出版社或主题相似但 DOI 不一致时，result 相关性为 0，不允许 LLM 猜测。 + +### 5.2 输入字段 + +| 输入 | 来源 | +|---|---| +| `query` | query 字段 | +| `title` | result 的 `title` 或 `display_name` | +| `abstract` | result 的 `abstract`、`summary` 或 `content` | +| `doi` | DOI query 的精确标识符匹配 | +| `unique_id`、`locations` | result 缺少顶层 DOI 时的标识符补充来源 | + +### 5.3 Result 级输出 + +| 字段 | 说明 | +|---|---| +| `relevance` | result 总相关性分数 | +| `query_relevance` | query 与 result 的语义匹配程度 | +| `result_quality` | result 内容质量辅助判断 | +| `content_issues` | LLM 判断是否存在内容问题 | +| `confidence` | LLM 对评分的置信度 | +| `error` | LLM 输出解析失败等错误 | +| `reasoning` | 简短原因 | + +DOI query 不调用 LLM。Result 级完全匹配为 `1.0`，否则为 `0.0`；reason 中记录 expected DOI 和 result DOI。 + +DOI 的 query 级得分按精确命中的排名折扣： + +```text +doi_relevance = max(exact_match / log2(rank + 1)) +``` + +因此 rank1 命中为 `1.0`，rank2 命中为 `0.63093`，没有精确命中为 `0.0`。普通 query 继续使用 top-k rank-discount mean。 + +### 5.4 Query 级异常 + +如果某个 query 的任意 rank 出现 LLM JSON 解析失败，会增加： + +```text +QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR +``` + +这类 label 表示运行/解析质量告警，不一定代表业务相关性低。分析低相关时建议区分： + +- `SEARCH_RESULT_RELEVANCE_LOW`：业务低相关。 +- `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`：LLM 输出格式或解析异常。 + +### 5.5 可调参数 + +| 参数 | 默认值 | 说明 | +|---|---:|---| +| `--top-k` | 10 | 每个 query 评测前 k 条结果 | +| `--threshold` | 0.15 | query 级 hard bad 阈值 | +| `--llm-max-tokens` | 1024 | LLM 输出最大 token 数 | +| `--llm-workers` | 4 | 并发 LLM 调用数 | +| `--llm-timeout` | 60 | 单次 LLM 请求超时秒数 | +| `--prompt-mode` | `detailed` | prompt 模式 | +| `OPENAI_MODEL` | `gpt-4o` | LLM 模型名，可通过环境变量覆盖 | +| `OPENAI_BASE_URL` | 空 | OpenAI compatible endpoint | +| `OPENAI_TEMPERATURE` | 0.0 | LLM temperature | + +### 5.6 LLM 模型选择建议 + +全量检索评测需要逐条判断 query-result 对，LLM 调用量通常接近 `query 数 × top-k`。建议优先使用低延迟的 Flash 类模型，例如： + +```text +OPENAI_MODEL=deepseek-v4-flash +``` + +- **全量评测和日常回归**：推荐 Flash 类模型，可显著缩短相关性判断时间，并降低长时间批处理中的超时风险。 +- **疑难样本复核**：Pro 类模型更适合抽取少量低分、边界或争议样本进行人工辅助复核，不建议直接用于数百至数千条 result 的常规全量评测。 +- **新旧版本对比**：两次评测必须固定相同模型、prompt、`max_tokens` 和 temperature；建议设置 `OPENAI_TEMPERATURE=0`，减少 LLM 随机波动。 +- **并发设置**：建议从 `--llm-workers 2` 至 `4` 开始，根据模型服务的限流和稳定性逐步调整。并发过高可能增加 5xx、超时或空响应。 + +模型名称由实际 OpenAI-compatible 服务决定，`deepseek-v4-flash` 仅作为当前环境的推荐示例，不是 Dingo 的强制依赖。 + +## 6. 内容有效性 Effectiveness + +### 6.1 业务逻辑 + +内容有效性判断的是一条检索结果记录是否“可读、完整、可用于用户判断”，不判断它是否与 query 相关。 + +当前不按 `metadata_type` 做差异化处理。也就是说，`paper`、`ebook`、未来新增类型都使用相同的 title、abstract、keywords、author 完整性标准。Venue 是否存在及其来源可信度统一交给权威性指标，避免对 ebook 等类型重复惩罚。 + +### 6.2 字段权重 + +单条 result 的分数为： + +```text +Effectiveness = + title_score * 0.30 ++ abstract_score * 0.50 ++ keywords_score * 0.10 ++ author_score * 0.10 +``` + +| 子项 | 权重 | 业务含义 | +|---|---:|---| +| `title_score` | 0.30 | 标题是否存在、是否可读 | +| `abstract_score` | 0.50 | 摘要是否存在、是否可读 | +| `keywords_score` | 0.10 | 是否至少存在一个非空关键词 | +| `author_score` | 0.10 | 是否至少存在一个非空作者值；不按作者数量额外加分 | + +字段为空时，该字段直接得 0 分。 + +示例：如果 result 只有非空且无异常的标题，其他字段为空，则 `title_score=1.0`： + +```text +1.0 * 0.30 + 0 + 0 + 0 = 0.30 +``` + +### 6.3 字段评分逻辑 + +每个字段先做基础质量判断： + +- 为空：0 分。 +- title、abstract、keywords、author 四个字段参与有效性评分并进行缺失检查。venue 不参与有效性加权，也不会因为缺失产生问题；其来源可信度由权威性指标负责。 +- title、abstract、keywords、venue、author 五个字段都会进行异常检查。venue 有值时仍检查 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声。HTML 标签和 Unicode 替换字符 `�` 出现即进入 LLM 二次确认；其他异常字符按规则阈值筛选。乱码筛选还包括 UTF-8 被误按 Latin-1 解码产生的 `Ð...`、`Ñ...` 序列及 C1 控制字符。 +- 字段非空：基础分为 1 分，不因字符长度、token 数、关键词数量或作者数量增减。 +- 字段存在 HTML 泄漏、乱码、不可见字符、不可读文本或严重特殊字符噪声：规则先召回候选，LLM 确认后降低相应字段分数。 +- 长文本不会获得额外加分，短文本也不会仅因长度被扣分。 + +`keywords` 只判断是否至少有一个非空值；空列表计为缺失，关键词数量不影响分数。 + +`venue` 读取优先级： + +```text +publication_venue_name_unified +publication_venue_name +venue +source +``` + +`author` 兼容 `author`、`authors` 字段以及字符串、对象、对象列表等常见结构。存在至少一个非空作者值时，作者基础分为 1；作者名称长度和作者数量不会影响基础分。乱码、HTML 或特殊字符噪声仍会进入异常检查。 + +对比不同检索后端时，需要把后端原始作者信息统一映射到 `author` 或 `authors`。未映射作者字段会被视为缺失并使单条 result 的有效性总分降低 `0.10`。 + +### 6.4 Issues 类型 + +| issue | 含义 | +|---|---| +| `missing_title` | 标题为空 | +| `missing_abstract` | 摘要为空 | +| `missing_keywords` | 关键词为空 | +| `missing_author` | 作者为空 | +| `title:html_tag` / `abstract:html_tag` / `venue:html_tag` / `author:html_tag` | LLM 判断字段中有 HTML/XML 标签泄漏 | +| `*:mojibake` | LLM 判断字段存在乱码或编码错误 | +| `*:invisible_char` | LLM 判断字段存在不可见/控制字符 | +| `*:unreadable_text` | LLM 判断字段整体不可读 | +| `*:special_char_noise` | LLM 判断字段中特殊字符噪声已经影响阅读 | +| `llm_quality_parse_error` | LLM 字段质量判断调用或解析失败 | + +注意： + +- `RuleSpecialCharacter` 和 `RuleInvisibleChar` 只用于快速召回疑似异常字段，不直接作为最终扣分依据。 +- LaTeX、化学符号、单位、希腊字母、`|` 分隔符等正常学术表达不应被 LLM 判为问题。 +- HTML 高亮标签、明显 mojibake、不可见字符、严重乱码会由 LLM 输出字段级 issue，并降低对应字段分数。 +- 分析 bad 样本时建议结合原始 title、abstract、keywords、venue、author 和 `llm_quality_reason` 进行人工抽查。 + +### 6.5 可调参数 + +| 参数 | 默认值 | 说明 | +|---|---:|---| +| `--top-k` | 10 | 每个 query 评测前 k 条结果 | +| `--threshold` | 0.15 | query 级 hard bad 阈值 | +| `--llm-max-tokens` | 1024 | 内容有效性 LLM 字段质量判断的最大 token 数 | +| `--llm-workers` | 4 | 内容有效性 LLM 二次复核并发数 | +| `--llm-timeout` | 60 | LLM 请求超时秒数 | +| `--disable-llm-quality` | false | 关闭 LLM 字段质量判断；保留字段缺失检查，并由规则候选直接触发异常扣分 | + +阈值建议： + +- `0.15` 适合作 hard bad，只筛严重不可用结果。 +- 如果要发现 metadata 缺失、摘要不足等一般质量问题，可额外关注 `< 0.45` 的 warning 区间。 + +## 7. 权威性 Authority + +### 7.1 业务逻辑 + +权威性判断检索结果是否具备学术可信度和影响力信号，主要来自： + +- 引用数。 +- 高影响引用数。 +- 期刊/会议/来源类型。 +- DOI。 + +该指标不判断 query 相关性，也不判断内容字段是否完整。它适合作为 overall 的辅助指标，不建议单独用于硬判“结果错误”。 + +### 7.2 字段权重 + +单条 result 的权威性分数为： + +```text +authority = + 0.45 * citation_score ++ 0.20 * influential_citation_score ++ 0.25 * venue_score ++ 0.10 * doi_score +``` + +| 子项 | 权重 | 业务含义 | +|---|---:|---| +| `citation_score` | 0.45 | 普通引用影响力 | +| `influential_citation_score` | 0.20 | 高影响引用 | +| `venue_score` | 0.25 | 来源/期刊/会议可信度 | +| `doi_score` | 0.10 | 是否具备 DOI 标识 | + +### 7.3 Citation 归一化 + +引用数使用 log 归一化，避免高引用老论文过度碾压。 + +```text +citation_score = log(1 + citation_count) / log(1 + 500) + +influential_citation_score = + log(1 + influential_citation_count) / log(1 + 50) +``` + +分数会被限制在 `[0, 1]`。 + +### 7.4 Venue 评分 + +`venue` 读取优先级： + +```text +publication_venue_name_unified +publication_venue_name +venue +source +``` + +当前规则： + +| 条件 | `venue_score` | reason | +|---|---:|---| +| 已知 repository 或 preprint（优先判断） | 0.45 | `repository_or_preprint` | +| 明确的学术 book series 或 ebook platform（优先判断） | 0.55 | `academic_book_series` | +| 明确命中权威期刊/会议家族 | 0.85 | `prestigious_venue_family` | +| 命中正规学术出版机构或来源组织 | 0.75 | `recognized_scholarly_publisher_or_venue` | +| `publication_venue_type` 是 journal/conference，或存在有效 ISSN | 0.65 | `structured_journal_or_conference` | +| 只有非空来源名称 | 0.40 | `named_venue` | +| 未知或低信号来源 | 0.25 | `unknown_or_low_signal_venue` | + +权威期刊家族使用带边界的规则匹配，包括： + +```text +Nature 及 Nature 学科子刊、Nature Communications、npj 系列、 +Communications 系列、Scientific Reports/Data、Science 官方期刊家族、 +部分 Cell Press 旗舰刊、NEJM、Lancet、JAMA、The BMJ、PNAS、 +JACS、PRL 以及主要 AI 会议 +``` + +匹配前会移除 HTML 高亮标签，并按 `|` 拆分中英文来源别名。规则不再使用无限制的普通子串：`Science Translational Medicine` 会命中 Science 家族，而 `Chemical Engineering Science` 不会。 + +正规出版机构包括 Springer Nature、Elsevier、Wiley、Oxford University Press、Cambridge University Press、IEEE、ACM、ACS、RSC、IOP、BMJ、PLOS、Royal Society、De Gruyter、CRC Press、World Scientific 等。出版商只能提供正规学术来源信号，因此分数低于明确命中的旗舰期刊。 + +ISSN 和 journal/conference 类型是防止白名单漏判的主要兜底：专业期刊即使不在硬编码刊名列表中，只要具有结构化来源信息，也能获得 0.65，而不是被直接判成低信号来源。 + +### 7.5 DOI 评分 + +```text +doi_score = 1.0 +``` + +条件： + +- result 有 `doi` 字段；或 +- `locations` 中包含 `doi.org`。 + +否则： + +```text +doi_score = 0.0 +``` + +### 7.6 使用注意 + +权威性对以下 query 类型可能偏保守： + +- 人名，例如 `Michael Pecht`、`张文宏`。 +- 泛词，例如 `Jerry`、`pam`。 +- 书籍、访谈、百科式结果。 +- 缺少 citation、DOI、venue 元数据但实际有用的结果。 + +因此，权威性低不一定表示检索结果不相关，只表示该结果缺少学术权威信号。 + +## 8. 综合评分 Overall + +综合脚本将三个 query 级指标加权： + +```text +overall = + 0.7 * relevance ++ 0.2 * effectiveness ++ 0.1 * authority +``` + +默认权重： + +| 指标 | 权重 | +|---|---:| +| `relevance` | 0.7 | +| `effectiveness` | 0.2 | +| `authority` | 0.1 | + +业务含义： + +- 相关性是核心，因此权重最高。 +- 内容有效性次之，保证结果有足够元数据可读。 +- 权威性作为辅助，不让 citation/DOI 过度主导搜索体验。 + +综合评估会同时检查： + +- `overall` 是否低于 overall 阈值。 +- `relevance` 是否低于相关性阈值。 +- 是否存在 LLM 解析错误。 +- `effectiveness` 是否低于有效性阈值。 +- `authority` 是否低于权威性阈值。 + +## 9. 使用命令 + +以下命令均在项目根目录执行。 + +### 内置测试数据 + +仓库提供了 `test/data/test_search_result.jsonl`，用于在提交代码前快速验证单指标和综合评测流程。该文件包含3条 query、9条 result： + +| Query | 场景 | +|---|---| +| `BiMLP` | 正常学术论文结果 | +| `海带` | 元数据中包含检索高亮 HTML | +| `pam` | 字段稀疏的 ebook 结果 | + +综合 smoke test： + +```powershell +python examples/retrieval/sdk_eval_search_result.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_quality_smoke ` + --top-k 3 ` + --llm-max-tokens 1024 ` + --effectiveness-llm-max-tokens 1024 ` + --relevance-threshold 0.15 ` + --effectiveness-threshold 0.15 ` + --authority-threshold 0.15 ` + --overall-threshold 0.15 ` + --save-good +``` + +相关性、有效性和综合 smoke test 需要预先设置 OpenAI-compatible 环境变量；权威性是纯规则评测，不需要 LLM API。 + +综合脚本按评测对象拆分分类目录： + +```text +/ +├── bad/ +│ ├── query_level/ +│ │ └── QUALITY_BAD/ +│ │ ├── SEARCH_RESULT_RELEVANCE_LOW.jsonl +│ │ ├── SEARCH_RESULT_EFFECTIVENESS_LOW.jsonl +│ │ └── SEARCH_RESULT_AUTHORITY_LOW.jsonl +│ └── result_level/ +│ ├── Relevance/ +│ │ └── Error_Relevance_Low.jsonl +│ ├── Effectiveness/ +│ │ ├── Error_Effectiveness_Low.jsonl +│ │ └── Error_HTML_Tag.jsonl +│ └── Authority/ +│ ├── Error_Authority_Low.jsonl +│ ├── Error_Citation_Miss.jsonl +│ └── Error_DOI_Miss.jsonl +└── good/ # 仅使用 --save-good 时生成 + ├── query_level/ + └── result_level/ +``` + +- `query_level`：一个 query 的 top-k 聚合得分及其全部结果。 +- `result_level`：每一篇检索文献的原始数据和三个指标明细。 +- 同一 result 可以写入多个原因 label 文件；例如 Authority Low 可能同时进入 Citation Miss 和 DOI Miss。 + +### 9.1 单独跑相关性 + +```bash +export OPENAI_API_KEY="" +export OPENAI_BASE_URL="" +export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_TEMPERATURE="0" + +python examples/retrieval/sdk_eval_relevancy.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_relevancy_97q \ + --top-k 10 \ + --threshold 0.15 \ + --llm-max-tokens 1024 \ + --llm-workers 3 \ + --llm-timeout 60 \ + --save-good +``` + +Windows PowerShell 示例： + +```powershell +$env:OPENAI_API_KEY="" +$env:OPENAI_BASE_URL="" +$env:OPENAI_MODEL="deepseek-v4-flash" +$env:OPENAI_TEMPERATURE="0" + +python examples/retrieval/sdk_eval_relevancy.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_relevancy_97q ` + --top-k 10 ` + --threshold 0.15 ` + --llm-max-tokens 1024 ` + --llm-workers 3 ` + --llm-timeout 60 ` + --save-good +``` + +### 9.2 单独跑内容有效性 + +```bash +export OPENAI_API_KEY="" +export OPENAI_BASE_URL="" +export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_TEMPERATURE="0" + +python examples/retrieval/sdk_eval_effectiveness.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_effectiveness_97q \ + --top-k 10 \ + --threshold 0.15 \ + --llm-max-tokens 1024 \ + --llm-workers 8 \ + --llm-timeout 60 \ + --save-good +``` + +### 9.3 单独跑权威性 + +```bash +python examples/retrieval/sdk_eval_authority.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_authority_97q \ + --top-k 10 \ + --threshold 0.15 \ + --save-good +``` + +### 9.4 跑综合评分 + +```bash +export OPENAI_API_KEY="" +export OPENAI_BASE_URL="" +export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_TEMPERATURE="0" + +python examples/retrieval/sdk_eval_search_result.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_quality_97q \ + --top-k 10 \ + --relevance-threshold 0.15 \ + --effectiveness-threshold 0.15 \ + --authority-threshold 0.15 \ + --overall-threshold 0.15 \ + --llm-max-tokens 1024 \ + --effectiveness-llm-max-tokens 512 \ + --llm-timeout 60 \ + --save-good +``` + +## 10. 阈值解释 + +当前默认统一阈值为： + +```text +0.15 +``` + +该阈值的含义是 hard bad，即只标记严重问题： + +- 相关性几乎不匹配。 +- 内容字段严重缺失或不可读。 +- 权威信号极弱。 + +分析建议： + +| 分数段 | 建议解释 | +|---|---| +| `< 0.15` | hard bad，优先人工排查 | +| `0.15 - 0.30` | 低分边缘，适合抽样检查 | +| `0.30 - 0.45` | 一般质量问题，尤其适合看 metadata 缺失 | +| `>= 0.45` | 通常可接受，但仍需结合业务 query 类型 | + +不同指标的阈值敏感性不同： + +- 相关性 `0.15` 适合筛严重错召回；如果要分析一般低相关，可关注 `< 0.30`。 +- 内容有效性 `0.15` 很宽松；如果要推动元数据补全，可关注 `< 0.45`。 +- 权威性 `0.15` 不宜轻易提高太多，因为很多人名、书籍、访谈、非标准论文结果天然 citation/DOI/venue 信号弱。 + +## 11. 常见分析方法 + +### 11.1 查看 query 级低分 + +```powershell +Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | + Sort-Object {[double]$_.relevance} | + Select-Object -First 20 query,relevance,error_count,label +``` + +### 11.2 查看 result 级有效性 issues + +```powershell +Import-Csv outputs/search_result_effectiveness_97q/result_scores.csv | + Where-Object {$_.issues -ne ""} | + Select-Object query,rank,title,Effectiveness,issues +``` + +### 11.3 查看权威性低分原因 + +```powershell +Import-Csv outputs/search_result_authority_97q/result_scores.csv | + Sort-Object {[double]$_.authority} | + Select-Object -First 20 query,rank,title,authority,citation_score,influential_citation_score,venue_score,doi_score,reason +``` + +### 11.4 区分相关性低分和解析错误 + +```powershell +Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | + Where-Object {[double]$_.relevance -lt 0.15} | + Select-Object query,relevance,error_count,label +``` + +```powershell +Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | + Where-Object {[int]$_.error_count -gt 0} | + Select-Object query,relevance,error_count,label +``` + +## 12. 相关代码位置 + +评测器： + +- `dingo/model/llm/llm_search_result_relevance.py` +- `dingo/model/llm/llm_search_result_effectiveness.py` +- `dingo/model/llm/llm_search_result_authority.py` + +脚本： + +- `examples/retrieval/sdk_eval_relevancy.py` +- `examples/retrieval/sdk_eval_effectiveness.py` +- `examples/retrieval/sdk_eval_authority.py` +- `examples/retrieval/sdk_eval_search_result.py` +- `examples/retrieval/search_result_eval_utils.py` + +## 13. 已知注意事项 + +1. 相关性使用 LLM，temperature 大于 0 时，同一批数据重跑可能有轻微分数波动。 +2. LLM 相关性结果可能出现 JSON 解析失败，脚本会记录 `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`。 +3. 内容有效性当前不按 `metadata_type` 放宽 title、abstract、keywords、author 的字段要求；venue 缺失不再降低有效性分数，由权威性指标统一判断。 +4. 内容有效性使用 `RuleSpecialCharacter` / `RuleInvisibleChar` / `RuleMojibake` 做快速初筛，再用 LLM 二次确认 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声；正常公式、LaTeX、单位符号不应被扣分。 +5. 权威性低不一定表示结果不相关，可能只是 citation、DOI、venue 元数据不足。 diff --git a/docs/search_result_relevance_executor.md b/docs/search_result_relevance_executor.md new file mode 100644 index 00000000..12cef9b7 --- /dev/null +++ b/docs/search_result_relevance_executor.md @@ -0,0 +1,59 @@ +# Search Result Relevance Executor Notes + +`examples/retrieval/sdk_eval_relevancy.py` evaluates each query-result pair with `LLMSearchResultRelevance` through Dingo `LocalExecutor`. + +## Content Issues + +`Relevance.Error_Content_Issues` now uses a strict standard. It is intended for severe visible content corruption, not for ordinary search-result incompleteness. + +The LLM prompt asks `content_issues=true` only when the visible title/content has severe problems such as: + +- mojibake or garbled text +- raw HTML/XML tag residue +- parser residue that materially hurts readability +- invisible or control characters +- unreadable text + +The evaluator also applies a deterministic evidence filter after the LLM response. Even if the LLM returns `content_issues=true`, the final executor label `Relevance.Error_Content_Issues` is emitted only when the result text contains supporting evidence. + +The following should not by itself trigger `Relevance.Error_Content_Issues`: + +- missing abstract +- short snippet +- truncated preview +- title-only result, if the title is still readable enough to judge relevance + +In `EvalDetail.reason`, the output keeps both: + +- `raw_content_issues`: the original LLM boolean +- `content_issues`: the post-filtered boolean used for final labels +- `content_issue_evidence`: the matched evidence list + +This keeps the LLM signal available for analysis while preventing ordinary truncation or sparse metadata from making an otherwise relevant result bad. + +## Test Data + +The repository includes `test/data/test_search_result.jsonl` for local smoke tests. It contains three queries and nine results: + +| Query | Covered scenario | +|---|---| +| `BiMLP` | Normal academic paper results | +| `海带` | Search-highlight HTML in result metadata | +| `pam` | Sparse ebook results | + +With the OpenAI-compatible environment variables configured, run: + +For full-dataset evaluation, prefer a low-latency Flash model such as `deepseek-v4-flash`. A Pro model is better reserved for reviewing a small number of ambiguous samples because pointwise relevance evaluation makes approximately `query count x top-k` LLM calls. Keep the model and prompt fixed and use temperature `0` when comparing search versions. + +```powershell +$env:OPENAI_MODEL="deepseek-v4-flash" +$env:OPENAI_TEMPERATURE="0" + +python examples/retrieval/sdk_eval_relevancy.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_relevancy_smoke ` + --top-k 3 ` + --llm-max-tokens 1024 ` + --threshold 0.15 ` + --save-good +``` diff --git a/examples/retrieval/sdk_eval_authority.py b/examples/retrieval/sdk_eval_authority.py new file mode 100644 index 00000000..2c4a7495 --- /dev/null +++ b/examples/retrieval/sdk_eval_authority.py @@ -0,0 +1,103 @@ +"""Run search result authority evaluation through Dingo LocalExecutor.""" + +from __future__ import annotations +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 + +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 + +EVALUATOR_NAME = "LLMSearchResultAuthority" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result authority with Dingo executor.") + add_common_args(parser) + parser.add_argument("--threshold", type=float, default=0.15) + parser.add_argument("--max-workers", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument( + "--raw-output", + action="store_true", + help="Write executor records with raw data merged into top-level JSONL rows.", + ) + return parser.parse_args() + + +def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + for rank, result in enumerate(item["results"][:top_k], start=1): + row: dict[str, Any] = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count + + +def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + return { + "task_name": "search_result_authority", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.max_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": args.save_good, + "all_labels": True, + "raw": args.raw_output, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [{"name": EVALUATOR_NAME, "config": {"threshold": args.threshold}}], + } + ], + } + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / "meta_search_flattened_authority_input.jsonl" + total = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + summary = Executor.exec_map["local"](InputArgs(**build_input_data(args, flattened_path))).execute() + print(summary) + print(f"[Done] flattened_input={flattened_path.resolve()}") + print(f"[Done] executor_output={summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/sdk_eval_effectiveness.py b/examples/retrieval/sdk_eval_effectiveness.py new file mode 100644 index 00000000..8ce3f0d7 --- /dev/null +++ b/examples/retrieval/sdk_eval_effectiveness.py @@ -0,0 +1,125 @@ +"""Run search result effectiveness evaluation through Dingo LocalExecutor.""" + +from __future__ import annotations +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 + +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 + +EVALUATOR_NAME = "LLMSearchResultEffectiveness" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result effectiveness with Dingo executor.") + add_common_args(parser) + parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY")) + parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL")) + parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o")) + parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0"))) + parser.add_argument("--llm-max-tokens", type=int, default=512) + parser.add_argument("--llm-workers", type=int, default=4) + parser.add_argument("--llm-timeout", type=float, default=60.0) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument("--threshold", type=float, default=0.15) + parser.add_argument( + "--disable-llm-quality", + action="store_true", + help="Disable LLM second judgment for abnormal-character candidates.", + ) + parser.add_argument( + "--raw-output", + action="store_true", + help="Write executor records with raw data merged into top-level JSONL rows.", + ) + return parser.parse_args() + + +def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + for rank, result in enumerate(item["results"][:top_k], start=1): + row: dict[str, Any] = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count + + +def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + llm_config: dict[str, Any] = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "max_tokens": args.llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.threshold, + "enable_llm_quality": not args.disable_llm_quality, + } + return { + "task_name": "search_result_effectiveness", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.llm_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": args.save_good, + "all_labels": True, + "raw": args.raw_output, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [{"name": EVALUATOR_NAME, "config": llm_config}], + } + ], + } + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / "meta_search_flattened_effectiveness_input.jsonl" + total = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + input_data = build_input_data(args, flattened_path) + summary = Executor.exec_map["local"](InputArgs(**input_data)).execute() + print(summary) + print(f"[Done] flattened_input={flattened_path.resolve()}") + print(f"[Done] executor_output={summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/sdk_eval_relevancy.py b/examples/retrieval/sdk_eval_relevancy.py new file mode 100644 index 00000000..6f37306d --- /dev/null +++ b/examples/retrieval/sdk_eval_relevancy.py @@ -0,0 +1,124 @@ +"""Run search result relevancy evaluation through Dingo LocalExecutor.""" + +from __future__ import annotations +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 + +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 + +EVALUATOR_NAME = "LLMSearchResultRelevance" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result relevancy with Dingo executor.") + add_common_args(parser) + parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY")) + parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL")) + parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o")) + parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0"))) + parser.add_argument("--prompt-mode", choices=("standard", "detailed"), default="detailed") + parser.add_argument("--expected-criteria", default=None) + parser.add_argument("--llm-max-tokens", type=int, default=1024) + parser.add_argument("--llm-workers", type=int, default=4) + parser.add_argument("--llm-timeout", type=float, default=60.0) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument("--threshold", type=float, default=0.15) + parser.add_argument( + "--raw-output", + action="store_true", + help="Write executor records with raw data merged into top-level JSONL rows.", + ) + return parser.parse_args() + + +def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + for rank, result in enumerate(item["results"][:top_k], start=1): + result_payload = dict(result) + result_payload["_eval_query"] = query + row: dict[str, Any] = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result_payload, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count + + +def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + llm_config: dict[str, Any] = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "prompt_mode": args.prompt_mode, + "expected_criteria": args.expected_criteria, + "max_tokens": args.llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.threshold, + } + return { + "task_name": "search_result_relevancy", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.llm_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": args.save_good, + "all_labels": True, + "raw": args.raw_output, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [{"name": EVALUATOR_NAME, "config": llm_config}], + } + ], + } + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / "meta_search_flattened_relevancy_input.jsonl" + total = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + summary = Executor.exec_map["local"](InputArgs(**build_input_data(args, flattened_path))).execute() + print(summary) + print(f"[Done] flattened_input={flattened_path.resolve()}") + print(f"[Done] executor_output={summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py new file mode 100644 index 00000000..b7488b5e --- /dev/null +++ b/examples/retrieval/sdk_eval_search_result.py @@ -0,0 +1,554 @@ +"""Evaluate search results with relevance, effectiveness, and authority metrics. + +This script uses Dingo LocalExecutor for result-level metric execution, then +aggregates executor outputs back to query-level CSV/JSON reports. +""" + +from __future__ import annotations +import argparse +import json +import math +import os +import sys +import time +from datetime import datetime +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json # noqa: E402,E501 + +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 +from dingo.model.llm.llm_search_result_relevance import is_doi_query # noqa: E402 + +WEIGHTS = { + "relevance": 0.7, + "effectiveness": 0.2, + "authority": 0.1, +} + +EFFECTIVENESS_LABEL_TO_ISSUE = { + "Effectiveness.Error_Title_Miss": "missing_title", + "Effectiveness.Error_Abstract_Miss": "missing_abstract", + "Effectiveness.Error_Keywords_Miss": "missing_keywords", + "Effectiveness.Error_Author_Miss": "missing_author", + "Effectiveness.Error_HTML_Tag": "html_tag", + "Effectiveness.Error_Mojibake": "mojibake", + "Effectiveness.Error_Invisible_Char": "invisible_char", + "Effectiveness.Error_Unreadable_Text": "unreadable_text", + "Effectiveness.Error_Special_Char_Noise": "special_char_noise", + "Effectiveness.Error_LLM_Quality_Parse": "llm_quality_parse_error", + "Effectiveness.Error_Effectiveness_Low": "effectiveness_low", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result quality with Dingo executor.") + add_common_args(parser) + parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY")) + parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL")) + parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o")) + parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0"))) + parser.add_argument("--prompt-mode", choices=("standard", "detailed"), default="detailed") + parser.add_argument("--llm-max-tokens", type=int, default=1024) + parser.add_argument("--llm-timeout", type=float, default=60.0) + parser.add_argument("--llm-workers", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument("--effectiveness-llm-max-tokens", type=int, default=512) + parser.add_argument( + "--disable-effectiveness-llm-quality", + action="store_true", + help="Disable LLM readability/corruption judgment for effectiveness.", + ) + parser.add_argument("--relevance-threshold", type=float, default=0.15) + parser.add_argument("--effectiveness-threshold", type=float, default=0.15) + parser.add_argument("--authority-threshold", type=float, default=0.15) + parser.add_argument("--overall-threshold", type=float, default=0.15) + parser.add_argument( + "--save-detailed", + action="store_true", + help="Save detailed_results.json with query-level records and embedded result rows.", + ) + return parser.parse_args() + + +def flatten_query_results( + input_path: Path, + output_path: Path, + *, + top_k: int, + max_queries: int | None, +) -> tuple[int, list[str]]: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + empty_queries: list[str] = [] + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + if not item["results"][:top_k]: + empty_queries.append(query) + for rank, result in enumerate(item["results"][:top_k], start=1): + result_payload = dict(result) + result_payload["_eval_query"] = query + row = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result_payload, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count, empty_queries + + +def build_executor_input(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + relevance_config = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "prompt_mode": args.prompt_mode, + "max_tokens": args.llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.relevance_threshold, + } + effectiveness_config = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "max_tokens": args.effectiveness_llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.effectiveness_threshold, + "enable_llm_quality": not args.disable_effectiveness_llm_quality, + } + authority_config = {"threshold": args.authority_threshold} + return { + "task_name": "search_result_quality", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.llm_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": True, + "all_labels": True, + "merge": True, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [ + {"name": "LLMSearchResultRelevance", "config": relevance_config}, + {"name": "LLMSearchResultEffectiveness", "config": effectiveness_config}, + {"name": "LLMSearchResultAuthority", "config": authority_config}, + ], + } + ], + } + + +def load_executor_records(executor_output_path: str) -> list[dict[str, Any]]: + all_results_path = Path(executor_output_path) / "all_results.jsonl" + if not all_results_path.exists(): + raise FileNotFoundError(f"Executor merged result file not found: {all_results_path}") + records = [] + with all_results_path.open("r", encoding="utf-8") as f: + for line in f: + if line.strip(): + records.append(json.loads(line)) + return records + + +def get_metric_detail(record: dict[str, Any], metric: str) -> dict[str, Any]: + details = record.get("eval_details", {}).get("search_result", []) + for detail in details: + if detail.get("metric") == metric: + return detail + return {} + + +def first_reason(detail: dict[str, Any]) -> dict[str, Any]: + reason = detail.get("reason") or [] + if reason and isinstance(reason[0], dict): + return reason[0] + return {} + + +def filtered_effectiveness_issues(detail: dict[str, Any]) -> str: + labels = detail.get("label") or [] + issues = [] + for label in labels: + issue = EFFECTIVENESS_LABEL_TO_ISSUE.get(label) + if issue and issue not in issues: + issues.append(issue) + return "|".join(issues) + + +def build_reports( + records: list[dict[str, Any]], + args: argparse.Namespace, + executor_summary, + executor_result_summary: dict[str, Any] | None = None, + empty_queries: list[str] | None = None, +) -> tuple[dict, list[dict], list[dict], list[dict], list[dict]]: + records = sorted( + records, + key=lambda r: ( + int(r.get("raw_data", {}).get("query_index") or 0), + int(r.get("raw_data", {}).get("rank") or 0), + ), + ) + + result_rows = [] + by_query: dict[str, list[dict[str, Any]]] = {} + rank_relevance_error_count = 0 + rank_effectiveness_llm_quality_error_count = 0 + + for record in records: + raw = record.get("raw_data", {}) + query = raw.get("query", "") + relevance_detail = get_metric_detail(record, "LLMSearchResultRelevance") + effectiveness_detail = get_metric_detail(record, "LLMSearchResultEffectiveness") + authority_detail = get_metric_detail(record, "LLMSearchResultAuthority") + relevance_reason = first_reason(relevance_detail) + effectiveness_reason = first_reason(effectiveness_detail) + authority_reason = first_reason(authority_detail) + + relevance = round(float(relevance_detail.get("score") or 0.0), 5) + effectiveness = round(float(effectiveness_detail.get("score") or 0.0), 5) + authority = round(float(authority_detail.get("score") or 0.0), 5) + overall = round( + WEIGHTS["relevance"] * relevance + + WEIGHTS["effectiveness"] * effectiveness + + WEIGHTS["authority"] * authority, + 5, + ) + + relevance_error = str(relevance_reason.get("error") or "") + effectiveness_error = str(effectiveness_reason.get("llm_quality_error") or "") + if relevance_error: + rank_relevance_error_count += 1 + if effectiveness_error: + rank_effectiveness_llm_quality_error_count += 1 + + row = { + "query": query, + "rank": raw.get("rank"), + "title": raw.get("title", ""), + "relevance": relevance, + "query_relevance": round(float(relevance_reason.get("query_relevance") or 0.0), 5), + "result_quality": round(float(relevance_reason.get("result_quality") or 0.0), 5), + "relevance_content_issues": bool(relevance_reason.get("content_issues", False)), + "relevance_content_issue_evidence": "|".join(relevance_reason.get("content_issue_evidence") or []), + "relevance_error": relevance_error, + "relevance_reasoning": relevance_reason.get("reasoning", ""), + "effectiveness": effectiveness, + "effectiveness_issues": filtered_effectiveness_issues(effectiveness_detail), + "effectiveness_llm_quality_reason": effectiveness_reason.get("llm_quality_reason", ""), + "effectiveness_llm_quality_error": effectiveness_error, + "authority": authority, + "citation_score": round(float(authority_reason.get("citation_score") or 0.0), 5), + "influential_citation_score": round(float(authority_reason.get("influential_citation_score") or 0.0), 5), + "venue_score": round(float(authority_reason.get("venue_score") or 0.0), 5), + "doi_score": round(float(authority_reason.get("doi_score") or 0.0), 5), + "authority_reason": authority_reason.get("reason", ""), + "overall": overall, + } + result_rows.append(row) + by_query.setdefault(query, []).append(row) + + query_rows = [] + detailed = [] + classified_records = [] + for query, rows in by_query.items(): + relevance_scores = [float(row["relevance"]) for row in rows] + effectiveness_scores = [float(row["effectiveness"]) for row in rows] + authority_scores = [float(row["authority"]) for row in rows] + if is_doi_query(query): + query_relevance = round( + max( + ( + float(row["relevance"]) + / math.log2(max(1, int(row.get("rank") or index)) + 1) + ) + for index, row in enumerate(rows, start=1) + ), + 5, + ) + relevance_aggregation = "doi_exact_match_rank_discount" + else: + query_relevance = round(rank_discounted_mean(relevance_scores), 5) + relevance_aggregation = "rank_discounted_mean" + query_effectiveness = round(rank_discounted_mean(effectiveness_scores), 5) + query_authority = round(rank_discounted_mean(authority_scores), 5) + query_overall = round( + WEIGHTS["relevance"] * query_relevance + + WEIGHTS["effectiveness"] * query_effectiveness + + WEIGHTS["authority"] * query_authority, + 5, + ) + + labels = [] + if query_overall < args.overall_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_OVERALL_LOW") + if query_relevance < args.relevance_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW") + relevance_errors = sum(1 for row in rows if row["relevance_error"]) + if relevance_errors: + labels.append("QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR") + if query_effectiveness < args.effectiveness_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW") + if query_authority < args.authority_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW") + + eval_status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD.SEARCH_RESULT_OVERALL_PASS"] + + query_row = { + "query": query, + "result_count": len(rows), + "valid_relevance_count": len(rows) - relevance_errors, + "relevance_error_count": relevance_errors, + "relevance": query_relevance, + "relevance_aggregation": relevance_aggregation, + "effectiveness": query_effectiveness, + "authority": query_authority, + "overall": query_overall, + "eval_status": eval_status, + "label": "|".join(labels), + } + query_rows.append(query_row) + detail = {**query_row, "results": rows} + detailed.append(detail) + classified_records.append({ + "query": query, + "metric": "search_result_quality", + "score": query_overall, + "threshold": args.overall_threshold, + "eval_status": eval_status, + "labels": labels, + "relevance": query_relevance, + "relevance_aggregation": relevance_aggregation, + "effectiveness": query_effectiveness, + "authority": query_authority, + "relevance_error_count": relevance_errors, + "thresholds": { + "relevance": args.relevance_threshold, + "effectiveness": args.effectiveness_threshold, + "authority": args.authority_threshold, + "overall": args.overall_threshold, + }, + "results": rows, + }) + + for query in empty_queries or []: + labels = ["QUALITY_BAD.SEARCH_RESULT_EMPTY"] + query_row = { + "query": query, + "result_count": 0, + "valid_relevance_count": 0, + "relevance_error_count": 0, + "relevance": 0.0, + "relevance_aggregation": ( + "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean" + ), + "effectiveness": 0.0, + "authority": 0.0, + "overall": 0.0, + "eval_status": True, + "label": labels[0], + } + query_rows.append(query_row) + detailed.append({**query_row, "results": []}) + classified_records.append({ + "query": query, + "metric": "search_result_quality", + "score": 0.0, + "threshold": args.overall_threshold, + "eval_status": True, + "labels": labels, + "relevance": 0.0, + "relevance_aggregation": ( + "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean" + ), + "effectiveness": 0.0, + "authority": 0.0, + "relevance_error_count": 0, + "thresholds": { + "relevance": args.relevance_threshold, + "effectiveness": args.effectiveness_threshold, + "authority": args.authority_threshold, + "overall": args.overall_threshold, + }, + "results": [], + }) + + summary = { + "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "metric": "search_result_quality", + "top_k": args.top_k, + "weights": WEIGHTS, + "thresholds": { + "relevance": args.relevance_threshold, + "effectiveness": args.effectiveness_threshold, + "authority": args.authority_threshold, + "overall": args.overall_threshold, + }, + "llm": { + "model": args.openai_model, + "prompt_mode": args.prompt_mode, + "max_tokens": args.llm_max_tokens, + "temperature": args.openai_temperature, + "timeout": args.llm_timeout, + "workers": args.llm_workers, + "effectiveness_llm_quality_enabled": not args.disable_effectiveness_llm_quality, + "effectiveness_llm_max_tokens": args.effectiveness_llm_max_tokens, + }, + "run_output_path": str(executor_summary.output_path), + "metrics": { + "relevance": summarize([float(row["relevance"]) for row in query_rows]), + "effectiveness": summarize([float(row["effectiveness"]) for row in query_rows]), + "authority": summarize([float(row["authority"]) for row in query_rows]), + "overall": summarize([float(row["overall"]) for row in query_rows]), + }, + "query_count": len(query_rows), + "result_count": len(result_rows), + "rank_relevance_error_count": rank_relevance_error_count, + "rank_effectiveness_llm_quality_error_count": rank_effectiveness_llm_quality_error_count, + "num_bad": sum(1 for row in query_rows if row["eval_status"]), + "num_good": sum(1 for row in query_rows if not row["eval_status"]), + } + if executor_result_summary: + summary["result_level"] = { + "score": executor_result_summary.get("score"), + "num_good": executor_result_summary.get("num_good"), + "num_bad": executor_result_summary.get("num_bad"), + "total": executor_result_summary.get("total"), + "type_ratio": executor_result_summary.get("type_ratio", {}), + "metrics_score": executor_result_summary.get("metrics_score", {}), + } + return summary, query_rows, result_rows, detailed, classified_records + + +def normalize_executor_summary(executor_output_path: str, records: list[dict[str, Any]]) -> dict[str, Any] | None: + """Make executor summary label ratios use record-level quality semantics. + + LocalExecutor counts every EvalDetail label. In this combined script one + result has three metrics, so a bad result can still contain QUALITY_GOOD + from the metrics that passed. This rewrite keeps error labels as + per-result occurrence rates, but makes QUALITY_GOOD mutually exclusive. + """ + total = len(records) + if total == 0: + return None + + field_key = "search_result" + counts: dict[str, int] = {} + for record in records: + labels = set() + for detail in record.get("eval_details", {}).get(field_key, []): + for label in detail.get("label") or []: + labels.add(label) + + if record.get("eval_status"): + labels.discard("QUALITY_GOOD") + else: + labels = {"QUALITY_GOOD"} + + for label in labels: + counts[label] = counts.get(label, 0) + 1 + + summary_path = Path(executor_output_path) / "summary.json" + if not summary_path.exists(): + return None + + summary = json.loads(summary_path.read_text(encoding="utf-8-sig")) + summary.setdefault("type_ratio", {})[field_key] = { + label: round(count / total, 6) + for label, count in sorted(counts.items()) + } + return summary + + +def build_result_classified_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Expose executor result labels for result-level directory output.""" + classified: list[dict[str, Any]] = [] + for record in records: + labels: list[str] = [] + for details in record.get("eval_details", {}).values(): + for detail in details: + for label in detail.get("label") or []: + if label != "QUALITY_GOOD" and label not in labels: + labels.append(label) + + eval_status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + classified.append({**record, "eval_status": eval_status, "labels": labels}) + return classified + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / f".search_result_quality_input_{int(time.time())}_{os.getpid()}.jsonl" + try: + total, empty_queries = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + input_data = build_executor_input(args, flattened_path) + executor_summary = Executor.exec_map["local"](InputArgs(**input_data)).execute() + run_dir = Path(executor_summary.output_path) + executor_records = load_executor_records(executor_summary.output_path) + executor_result_summary = normalize_executor_summary(executor_summary.output_path, executor_records) + summary, query_rows, result_rows, detailed, classified_records = build_reports( + executor_records, + args, + executor_summary, + executor_result_summary, + empty_queries, + ) + + write_json(run_dir / "summary.json", summary) + if args.save_detailed: + write_json(run_dir / "detailed_results.json", {"summary": summary, "queries": detailed}) + write_csv(run_dir / "query_scores.csv", query_rows) + write_csv(run_dir / "result_scores.csv", result_rows) + write_classified_jsonl( + run_dir, + classified_records, + save_good=args.save_good, + level="query_level", + ) + write_classified_jsonl( + run_dir, + build_result_classified_records(executor_records), + save_good=args.save_good, + level="result_level", + ) + print(f"Saved to {run_dir.resolve()}") + finally: + if flattened_path.exists(): + flattened_path.unlink() + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/search_result_eval_utils.py b/examples/retrieval/search_result_eval_utils.py new file mode 100644 index 00000000..d4189d4f --- /dev/null +++ b/examples/retrieval/search_result_eval_utils.py @@ -0,0 +1,117 @@ +"""Utilities for search result JSONL evaluation examples.""" + +from __future__ import annotations +import argparse +import csv +import json +import math +import statistics +from pathlib import Path +from typing import Any + + +def load_query_result_jsonl(path: Path, max_queries: int | None = None) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8-sig") as f: + for line_no, line in enumerate(f, start=1): + if not line.strip(): + continue + item = json.loads(line) + query = item.get("query") or item.get("query_text") or item.get("q") or "" + results = ( + item.get("results") + or item.get("top_results") + or item.get("top_api_results") + or item.get("search_results") + or [] + ) + if isinstance(results, dict): + results = results.get("results") or results.get("items") or [] + if not isinstance(results, list): + raise ValueError(f"Line {line_no}: results must be a list.") + if not query: + raise ValueError(f"Line {line_no}: missing query/query_text.") + items.append({**item, "query": str(query), "results": results}) + if max_queries and len(items) >= max_queries: + break + return items + + +def get_title(result: dict[str, Any]) -> str: + return str(result.get("title") or result.get("display_name") or "") + + +def rank_discounted_mean(values: list[float]) -> float: + if not values: + return 0.0 + weights = [1.0 / math.log2(rank + 2) for rank in range(len(values))] + return sum(value * weight for value, weight in zip(values, weights)) / sum(weights) + + +def summarize(values: list[float]) -> dict[str, float | int]: + if not values: + return {"count": 0, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0} + return { + "count": len(values), + "mean": round(statistics.mean(values), 5), + "median": round(statistics.median(values), 5), + "min": round(min(values), 5), + "max": round(max(values), 5), + } + + +def add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--input-jsonl", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, default=Path("outputs/search_result_eval")) + parser.add_argument("--top-k", type=int, default=10) + parser.add_argument("--max-queries", type=int, default=None) + parser.add_argument("--save-good", action="store_true", help="Save good query-level records under good/.") + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8-sig") + return + fieldnames: list[str] = [] + for row in rows: + for key in row: + if key not in fieldnames: + fieldnames.append(key) + with path.open("w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def write_classified_jsonl( + output_dir: Path, + records: list[dict[str, Any]], + *, + save_good: bool = False, + level: str | None = None, +) -> None: + for record in records: + status = "bad" if record.get("eval_status") else "good" + if status == "good" and not save_good: + continue + labels = record.get("labels") or [] + if not labels: + labels = ["QUALITY_GOOD.PASS"] if status == "good" else ["QUALITY_BAD.UNKNOWN"] + for label in labels: + parts = str(label).split(".") + status_dir = output_dir / status + if level: + status_dir = status_dir / level + if len(parts) > 1: + path = status_dir / Path(*parts[:-1]) / f"{parts[-1]}.jsonl" + else: + path = status_dir / f"{parts[0]}.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") diff --git a/test/data/test_search_result.jsonl b/test/data/test_search_result.jsonl new file mode 100644 index 00000000..c39ad0e3 --- /dev/null +++ b/test/data/test_search_result.jsonl @@ -0,0 +1,3 @@ +{"query":"BiMLP","results":[{"type":["preprint","article"],"publication_venue_type":"repository","access_license":"nonexclusive-distrib","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"bfc49e21253ac0f4278e66c0c59b0bcfe99cc8b30f1b5f1cde36d0e85b957b33","cited_by_percentile_year":{},"classifications":{"arxiv_category":["cs.CV"]},"citation_count":9.0,"influential_citation_count":0.0,"abstract":"This paper studies the problem of designing compact binary architectures for vision multi-layer perceptrons (MLPs). We provide extensive analysis on the difficulty of binarizing vision MLPs and find that previous binarization methods perform poorly due to limited capacity of binary MLPs. In contrast with the traditional CNNs that utilizing convolutional operations with large kernel size, fully-connected (FC) layers in MLPs can be treated as convolutional layers with kernel size $1\\times1$. Thus, the representation ability of the FC layers will be limited when being binarized, and places restrictions on the capability of spatial mixing and channel mixing on the intermediate features. To this end, we propose to improve the performance of binary MLP (BiMLP) model by enriching the representation ability of binary FC layers. We design a novel binary block that contains multiple branches to merge a series of outputs from the same stage, and also a universal shortcut connection that encourages the information flow from the previous stage. The downsampling layers are also carefully designed to reduce the computational complexity while maintaining the classification performance. Experimental results on benchmark dataset ImageNet-1k demonstrate the effectiveness of the proposed BiMLP models, which achieve state-of-the-art accuracy compared to prior binary CNNs. The MindSpore code is available at \\url{https://gitee.com/mindspore/models/tree/master/research/cv/BiMLP}.","publication_publisher":[],"access_oa_status":"green","relevance_score":26.327469,"is_content_accessible":true,"publication_venue_name_unified":"arXiv (Cornell University)","locations":[{"type":"","is_oa":"true","url":"http://arxiv.org/abs/2212.14158","license":""},{"type":"","is_oa":"true","url":"https://arxiv.org/pdf/2212.14158","license":""},{"type":"","is_oa":"true","url":"https://doi.org/10.48550/arxiv.2212.14158","license":""},{"type":"","is_oa":"unknown","url":"https://arxiv.org/abs/2212.14158","license":""},{"type":"download","is_oa":"true","url":"https://arxiv.org/pdf/2212.14158v1","license":"nonexclusive-distrib"},{"type":"download","is_oa":"true","url":"https://arxiv.org/src/2212.14158v1","license":"nonexclusive-distrib"}],"publication_published_year":2022.0,"keywords":["Computer science","Perceptron","Binary number","Benchmark (surveying)","Binary tree","Layer (electronics)","Artificial intelligence","Convolutional neural network","Kernel (algebra)","Merge (version control)","Pattern recognition (psychology)","Algorithm","Parallel computing","Artificial neural network","Mathematics"],"unique_id":"paper:10.48550/arxiv.2212.14158","doi":"10.48550/arxiv.2212.14158","access_is_oa":"true","doc_id":"bfc49e21253ac0f4278e66c0c59b0bcfe99cc8b30f1b5f1cde36d0e85b957b33","language":"en","citation_normalized_percentile":{},"access_oa_url":["https://arxiv.org/pdf/2212.14158","https://arxiv.org/pdf/2212.14158v1"],"publication_venue_issn":[],"reference_count":55.0,"title":"BiMLP: Compact Binary Architectures for Vision Multi-Layer Perceptrons","author":[{"orcid":"https://orcid.org/0000-0002-9704-4194","name":"Yixing Xu"},{"orcid":"https://orcid.org/0000-0002-2102-8235","name":"Xinghao Chen"},{"orcid":"https://orcid.org/0000-0002-0013-4530","name":"Yunhe Wang"}]},{"type":["article"],"publication_venue_type":"","access_license":"","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":[],"access_oa_status":"closed","relevance_score":24.286182,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[{"type":"","is_oa":"false","url":"https://doi.org/10.52202/068431-0367","license":""}],"publication_published_year":2022.0,"keywords":["Perceptron","Binary number","Binary data","Pattern recognition (psychology)","Artificial neural network","Feature (linguistics)"],"unique_id":"paper:10.52202/068431-0367","doi":"10.52202/068431-0367","access_is_oa":"false","language":"","citation_normalized_percentile":{"value":0.33077007,"is_in_top_10_percent":"false","is_in_top_1_percent":"false"},"access_oa_url":[],"publication_venue_issn":[],"title":"BiMLP: Compact Binary Architectures for Vision Multi-Layer Perceptrons","reference_count":0.0,"fwci":0.0,"author":[{"orcid":"","name":"Xinghao Chen"},{"orcid":"","name":"Yunhe Wang"},{"orcid":"","name":"Yixing Xu"}]},{"type":["article"],"publication_venue_type":"journal","access_license":"cc-by","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"Abstract. Satellite radar altimetry has significantly enhanced inland water monitoring by providing consistent, long-term lake-level observations. However, these datasets often contain substantial gaps caused by sensor malfunctions, orbital limitations, or retrieval errors, complicating hydrological analyses and downstream applications. This study introduces a robust benchmarking framework designed to systematically evaluate gap-filling techniques in satellite-derived lake water-level records through controlled pseudo-gap injection experiments. We investigated three large lakes with distinct hydrological behaviours, the Caspian Sea, Lake Superior, and Lake Tanganyika each representing unique temporal dynamics. Synthetic seven-year gaps (2002–2008) were artificially introduced into complete altimetry datasets, and three sophisticated gap-filling methods were compared: Singular Spectrum Analysis (SSA), Bidirectional Autoregressive (BIAR) models, and Bidirectional Multi-Layer Perceptrons (BiMLP). Method performance was assessed using standard metrics (RMSE, MAE, Bias, and R2), alongside statistical properties including variance, skewness, autocorrelation, and stationarity. BiMLP consistently delivered the highest accuracy across all study lakes, demonstrating exceptional adaptability to both smooth and highly variable signals. SSA performed effectively for lakes exhibiting quasi-periodic behavior, while BIAR showed sensitivity to lag selection and reduced performance under non-stationary conditions. These results emphasize the critical role of bidirectional modeling approaches and rigorous time-series diagnostics in selecting appropriate gap-filling methods for satellite altimetry-based hydrological studies. These findings provide practical guidance for selecting appropriate gap-filling strategies under long data outages in satellite altimetry workflows.","publication_publisher":["Copernicus Publications"],"access_oa_status":"diamond","relevance_score":8.976683,"is_content_accessible":false,"publication_venue_name_unified":"ISPRS annals of the photogrammetry, remote sensing and spatial information sciences","locations":[{"type":"","is_oa":"true","url":"https://doi.org/10.5194/isprs-annals-x-4-w8-2025-557-2026","license":"cc-by"},{"type":"","is_oa":"true","url":"https://isprs-annals.copernicus.org/articles/X-4-W8-2025/557/2026/isprs-annals-X-4-W8-2025-557-2026.pdf","license":"cc-by"},{"type":"","is_oa":"true","url":"https://doaj.org/article/72149efbb5e740b4a046e48e9a2acb4d","license":"cc-by-sa"}],"publication_published_year":2026.0,"keywords":["Benchmarking","Satellite","Sensitivity (control systems)","Adaptability","Water level","Altimeter","Autoregressive model","Satellite altimetry"],"unique_id":"paper:10.5194/isprs-annals-x-4-w8-2025-557-2026","doi":"10.5194/isprs-annals-x-4-w8-2025-557-2026","access_is_oa":"true","language":"en","citation_normalized_percentile":{"value":0.77162252,"is_in_top_10_percent":"false","is_in_top_1_percent":"false"},"access_oa_url":["https://isprs-annals.copernicus.org/articles/X-4-W8-2025/557/2026/isprs-annals-X-4-W8-2025-557-2026.pdf"],"publication_venue_issn":["2194-9042","2194-9050","2196-6346"],"title":"Benchmarking Gap-Filling Techniques in Satellite Altimetry-Based Lake Water-Level Time Series","reference_count":0.0,"fwci":0.0,"author":[{"orcid":"","name":"Ali MusaHoseini"},{"orcid":"https://orcid.org/0000-0002-0534-0632","name":"Saeed Farzaneh"},{"orcid":"https://orcid.org/0000-0003-3055-041X","name":"Ehsan Forootan"}]}]} +{"query":"海带","results":[{"type":[],"publication_venue_type":"","access_license":"","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"8b1c2021d710f1abf3a83f6dc28fabc6c5b19b57a8447286e232871113a23604","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"每次幼儿园吃海带,孩子们都不喜欢,端回的餐盘里都会剩下很多海带.鉴于老师说过“挑食不是好孩子”,所以他们不吃的借口也千奇百怪“老师,我已经吃饱了!”“老师我实在吃不下了!”“老师,我觉得这里面是青色的虫子,所以我……”哎!怎么做才能改善这种情况呢?","publication_publisher":[],"access_oa_status":"","relevance_score":32.074493,"is_content_accessible":true,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e教育\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e实践\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e与\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e研究\u003c/span\u003e | Educational Practice \u0026 Research","locations":[],"publication_published_year":2015.0,"keywords":["凉拌海带丝","《海带的传说》","海带","挑食"],"unique_id":"paper:10.3969/j.issn.1009-010x.2015.01.028","doi":"10.3969/j.issn.1009-010x.2015.01.028","access_is_oa":"unknown","doc_id":"8b1c2021d710f1abf3a83f6dc28fabc6c5b19b57a8447286e232871113a23604","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1009-010X"],"reference_count":0.0,"title":"海带的传说","author":[{"orcid":"","name":"卢瑞云"}]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"","access_xinghe_repository_sha256":"","metadata_type":"paper","classifications":{},"citation_count":0.0,"abstract":"宝宝,今天邀请爸爸妈妈玩\"海带拳\"游戏吧!宝宝当出拳人,妈妈当挑战者,两人一起喊:\"海带,海带!\"同时双手握拳,双臂上下左右挥舞.如果妈妈手臂方向和宝宝不一样,游戏继续;如果一样,挑战失败,换爸爸挑战宝宝.","publication_publisher":[],"access_oa_status":"","relevance_score":29.815878,"is_content_accessible":false,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e动漫\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e界\u003c/span\u003e | Dong Man Jie","locations":[],"publication_published_year":2020.0,"access_is_oa":"unknown","doi":"10.3969/j.issn.1673-8438.2020.43.009","keywords":[],"unique_id":"paper:10.3969/j.issn.1673-8438.2020.43.009","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1673-8438"],"title":"海带拳","reference_count":0.0,"author":[{"orcid":"","name":"深红"}]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"","access_xinghe_repository_sha256":"","metadata_type":"paper","classifications":{},"citation_count":0.0,"abstract":"育种单位:中国科学院海洋研究所、荣成市蜊江水产有限责任公司\r\n品种简介:该品种是以荣成海带栽培群体后代个体的雌配子体和韩国海带自然种群后代个体的雄配子体杂交产生的后代群体为亲本群体,以藻体深褐色、叶片宽大和孢子囊发育良好为选育指标,采用群体选育技术,经连续4代选育而成.\r\n在相同栽培条件下,与普通海带品种相比,在水温6℃左右(4月上旬)可开始收获,收获期可延续至水温达到19℃左右(7月中下旬),产量提高15.0%以上,抗高温、高光能力较强,淡干海带色泽墨绿.","publication_publisher":[],"access_oa_status":"","relevance_score":29.779959,"is_content_accessible":false,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e中国\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e水产\u003c/span\u003e | China Fisheries","locations":[],"publication_published_year":2015.0,"access_is_oa":"unknown","doi":"10.3969/j.issn.1002-6681.2015.10.029","keywords":[],"unique_id":"paper:10.3969/j.issn.1002-6681.2015.10.029","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1002-6681"],"title":"海带\"205\"","reference_count":0.0,"author":[{"orcid":"","name":"刘峰"},{"orcid":"","name":"王嘉琪"},{"orcid":"","name":"逄少军"},{"orcid":"","name":"刘启顺"},{"orcid":"","name":"孙长彬"}]}]} +{"query":"pam","results":[{"type":["printbook"],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9788362863390","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Pracownia Wydawnicza \"ElSet\""],"access_oa_status":"","relevance_score":27.211927,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":2013.0,"access_is_oa":"unknown","doi":"","keywords":[],"unique_id":"ebook:9788362863390","language":"pl","access_oa_url":[],"publication_venue_issn":[],"title":"Pam Pam Pam","reference_count":0.0,"author":[{"orcid":"","name":"Jerzy Szczudlik"}],"isbns":["9788362863396","9788362863390"]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9788362863396","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Pracownia Wydawnicza \"ElSet\""],"access_oa_status":"","relevance_score":27.09857,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":2013.0,"access_is_oa":"","doi":"","keywords":[],"unique_id":"ebook:9788362863396","language":"pl","access_oa_url":[],"publication_venue_issn":[],"title":"Pam Pam Pam","reference_count":0.0,"author":[{"orcid":"","name":"Jerzy Szczudlik"}],"isbns":["9788362863396"]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9780439397858","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Scholastic, Inc."],"access_oa_status":"","relevance_score":27.048462,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":1995.0,"access_is_oa":"unknown","doi":"","keywords":[],"unique_id":"ebook:9780439397858","language":"en","access_oa_url":[],"publication_venue_issn":[],"title":"Pam!, Pam!, Pam!","reference_count":0.0,"author":[{"orcid":"","name":"Eve Merriam"}],"isbns":["9780439397858"]}]} diff --git a/test/scripts/model/llm/test_llm_search_result_authority.py b/test/scripts/model/llm/test_llm_search_result_authority.py new file mode 100644 index 00000000..88d5633b --- /dev/null +++ b/test/scripts/model/llm/test_llm_search_result_authority.py @@ -0,0 +1,126 @@ +from dingo.model.llm.llm_search_result_authority import LLMSearchResultAuthority, _has_doi_in_locations + + +class NonSerializableLocation: + pass + + +def test_has_doi_in_location_dict_or_string(): + assert _has_doi_in_locations([{"url": "https://doi.org/10.1000/test"}]) + assert _has_doi_in_locations(["https://DOI.ORG/10.1000/test"]) + + +def test_has_doi_in_locations_ignores_invalid_shapes(): + assert not _has_doi_in_locations(True) + assert not _has_doi_in_locations({"url": "https://doi.org/10.1000/test"}) + assert not _has_doi_in_locations([NonSerializableLocation()]) + + +def test_authority_grade_handles_non_serializable_location(): + grade = LLMSearchResultAuthority().grade( + result={"doi": "", "locations": [NonSerializableLocation()]}, + ) + + assert grade.doi_score == 0.0 + + +def test_nature_portfolio_families_are_recognized(): + grader = LLMSearchResultAuthority() + + for venue in ( + "Nature Physics", + "Nature Communications", + "npj Digital Medicine", + "Communications Biology", + "Scientific Reports", + "Scientific Data", + ): + grade = grader.grade(result={"publication_venue_name_unified": venue}) + assert grade.venue_score == 0.85 + assert grade.reason == "prestigious_venue_family" + + +def test_generic_science_names_do_not_match_science_family(): + grader = LLMSearchResultAuthority() + + named_only = grader.grade(result={"publication_venue_name_unified": "Grand Garden of Science"}) + structured = grader.grade( + result={ + "publication_venue_name_unified": "Chemical Engineering Science", + "publication_venue_type": "journal", + } + ) + + assert named_only.venue_score == 0.4 + assert named_only.reason == "named_venue" + assert structured.venue_score == 0.65 + assert structured.reason == "structured_journal_or_conference" + + +def test_repository_takes_priority_over_name_patterns(): + grade = LLMSearchResultAuthority().grade( + result={"publication_venue_name_unified": "Open Science Framework"}, + ) + + assert grade.venue_score == 0.45 + assert grade.reason == "repository_or_preprint" + + +def test_issn_and_recognized_publisher_provide_venue_fallbacks(): + grader = LLMSearchResultAuthority() + issn_grade = grader.grade( + result={ + "publication_venue_name_unified": "Specialist Research Journal", + "publication_venue_issn": ["1234-567X"], + } + ) + publisher_grade = grader.grade( + result={ + "publication_venue_name_unified": "Specialist Research Journal", + "publication_publisher": ["Oxford University Press"], + } + ) + + assert issn_grade.venue_score == 0.65 + assert issn_grade.reason == "structured_journal_or_conference" + assert publisher_grade.venue_score == 0.75 + assert publisher_grade.reason == "recognized_scholarly_publisher_or_venue" + + +def test_html_highlight_is_removed_before_venue_matching(): + grade = LLMSearchResultAuthority().grade( + result={"publication_venue_name_unified": "Nature Medicine"}, + ) + + assert grade.venue_score == 0.85 + + +def test_ieee_is_recognized_inside_full_conference_name(): + grade = LLMSearchResultAuthority().grade( + result={ + "publication_venue_name_unified": ( + "13th IEEE International Workshops on Enabling Technologies" + ) + }, + ) + + assert grade.venue_score == 0.75 + assert grade.reason == "recognized_scholarly_publisher_or_venue" + + +def test_ebook_platform_and_academic_publisher_are_recognized(): + grader = LLMSearchResultAuthority() + ebook_grade = grader.grade( + result={ + "publication_venue_name_unified": "Springer eBooks", + "publication_venue_type": "ebook platform", + } + ) + publisher_grade = grader.grade( + result={"publication_publisher": ["Walter De Gruyter & Co"]}, + ) + + assert ebook_grade.venue_score == 0.55 + assert ebook_grade.reason == "academic_book_series" + assert publisher_grade.venue_score == 0.75 + assert publisher_grade.reason == "recognized_scholarly_publisher_or_venue" diff --git a/test/scripts/model/llm/test_llm_search_result_effectiveness.py b/test/scripts/model/llm/test_llm_search_result_effectiveness.py new file mode 100644 index 00000000..8001d2b8 --- /dev/null +++ b/test/scripts/model/llm/test_llm_search_result_effectiveness.py @@ -0,0 +1,183 @@ +import pytest + +from dingo.model.llm.llm_search_result_effectiveness import ( # isort: skip + LLMSearchResultEffectiveness, + _filter_llm_field_issues, + _issues_to_labels, + _looks_like_utf8_latin1_mojibake, + _rule_abnormal_char_issues, + extract_authors, +) + + +def _mojibake(value: str) -> str: + return value.encode("utf-8").decode("latin-1") + + +def test_detects_utf8_cyrillic_decoded_as_latin1(): + broken = _mojibake("Развитие научных исследований") + + assert _looks_like_utf8_latin1_mojibake(broken) + assert "RuleMojibake" in _rule_abnormal_char_issues(broken) + assert _filter_llm_field_issues("title", broken, ["title:mojibake"]) == ["title:mojibake"] + assert _issues_to_labels(["RuleMojibake", "title:mojibake"]) == ["Effectiveness.Error_Mojibake"] + + +def test_does_not_flag_valid_latin_or_cyrillic_text(): + assert not _looks_like_utf8_latin1_mojibake("Ð is a valid Icelandic letter") + assert not _looks_like_utf8_latin1_mojibake("Развитие научных исследований") + + +def test_detects_mojibake_fragment_in_mixed_language_text(): + broken = "中文标题 | " + _mojibake("Научные исследования") + + assert _looks_like_utf8_latin1_mojibake(broken) + + +def test_rule_only_grade_penalizes_mojibake_fields(): + broken_title = _mojibake("Развитие научных исследований") + broken_abstract = _mojibake( + "В этой статье рассматриваются современные научные исследования и методы анализа данных. " * 4 + ) + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + + grade = grader.grade( + title=broken_title, + abstract=broken_abstract, + keywords=["research", "analysis", "data"], + venue="Science Journal", + ) + + assert "RuleMojibake" in grade.issues + assert grade.title_score == 0.1 + assert grade.abstract_score == 0.1 + assert grade.score < 0.5 + + +def test_extract_authors_supports_common_response_shapes(): + assert extract_authors({"author": [{"name": "Alice"}, {"display_name": "张三"}]}) == ["Alice", "张三"] + assert extract_authors({"authors": "Alice | Bob"}) == ["Alice", "Bob"] + assert extract_authors({"author": {"author_name": "Carol"}}) == ["Carol"] + + +def test_author_is_scored_without_rewarding_author_count(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + common = { + "title": "A comprehensive evaluation of academic search result metadata", + "abstract": "academic search metadata provides useful information for readers " * 15, + "keywords": ["search", "metadata", "quality", "evaluation", "academic"], + "venue": "International Journal of Search Quality Research", + } + + single_author = grader.grade(**common, authors=["Alice"]) + multiple_authors = grader.grade(**common, authors=["Alice", "Bob", "Carol"]) + missing_author = grader.grade(**common) + + assert single_author.author_score == 1.0 + assert multiple_authors.author_score == 1.0 + assert single_author.score == multiple_authors.score + assert missing_author.author_score == 0.0 + assert missing_author.score == pytest.approx(single_author.score - 0.1) + assert "missing_author" in missing_author.issues + assert _issues_to_labels(missing_author.issues) == ["Effectiveness.Error_Author_Miss"] + + +def test_nonempty_fields_are_not_penalized_for_length_or_item_count(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + + grade = grader.grade( + title="D", + abstract="短", + keywords=["AI"], + venue="J", + authors=["Q"], + ) + + assert grade.title_score == 1.0 + assert grade.abstract_score == 1.0 + assert grade.keywords_score == 1.0 + assert grade.venue_score == 1.0 + assert grade.author_score == 1.0 + assert grade.score == 1.0 + assert grade.issues == [] + + +def test_longer_content_does_not_receive_more_effectiveness_credit(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + short = grader.grade(title="D", abstract="A", keywords=["K"], authors=["Q"]) + long = grader.grade( + title="A comprehensive academic title", + abstract="A complete and readable abstract. " * 100, + keywords=["one", "two", "three", "four", "five"], + authors=["Alice", "Bob"], + ) + + assert short.score == long.score == 1.0 + + +def test_missing_venue_is_diagnostic_only_and_does_not_reduce_score(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + common = { + "title": "A comprehensive evaluation of academic search result metadata", + "abstract": "academic search metadata provides useful information for readers " * 15, + "keywords": ["search", "metadata", "quality", "evaluation", "academic"], + "authors": ["Alice"], + } + + with_venue = grader.grade(**common, venue="International Journal of Search Quality Research") + without_venue = grader.grade(**common, venue="") + + assert with_venue.score == without_venue.score + assert without_venue.venue_score == 0.0 + assert "missing_venue" not in without_venue.issues + assert "Effectiveness.Error_Venue_Miss" not in _issues_to_labels(without_venue.issues) + + +def _complete_result() -> dict: + return { + "title": "A comprehensive evaluation of academic search result metadata", + "abstract": "academic search metadata provides useful information for readers " * 15, + "keywords": ["search", "metadata", "quality", "evaluation", "academic"], + "publication_venue_name_unified": "International Journal of Search Quality Research", + "author": [{"name": "Alice"}], + } + + +@pytest.mark.parametrize("field", ["title", "abstract", "keywords", "venue", "author"]) +def test_all_effectiveness_fields_scan_html_residue(field: str): + result = _complete_result() + contaminated = "clean text leaked markup" + if field == "keywords": + result["keywords"] = [contaminated] + elif field == "venue": + result["publication_venue_name_unified"] = contaminated + elif field == "author": + result["author"] = [{"name": contaminated}] + elif field == "abstract": + result["abstract"] = "readable abstract text " * 100 + contaminated + else: + result[field] = contaminated + + grade = LLMSearchResultEffectiveness(enable_llm_quality=False).grade(result=result) + + assert "RuleSpecialCharacter" in grade.issues + assert getattr(grade, f"{field}_score") <= 0.1 + + +@pytest.mark.parametrize("field", ["title", "abstract", "keywords", "venue", "author"]) +def test_all_effectiveness_fields_scan_replacement_character(field: str): + result = _complete_result() + contaminated = "metadata contains \ufffd broken text" + if field == "keywords": + result["keywords"] = [contaminated] + elif field == "venue": + result["publication_venue_name_unified"] = contaminated + elif field == "author": + result["author"] = [{"name": contaminated}] + else: + result[field] = contaminated + + grade = LLMSearchResultEffectiveness(enable_llm_quality=False).grade(result=result) + + assert "RuleMojibake" in grade.issues + assert getattr(grade, f"{field}_score") <= 0.1 diff --git a/test/scripts/model/llm/test_llm_search_result_relevance.py b/test/scripts/model/llm/test_llm_search_result_relevance.py new file mode 100644 index 00000000..210d6d67 --- /dev/null +++ b/test/scripts/model/llm/test_llm_search_result_relevance.py @@ -0,0 +1,48 @@ +from dingo.model.llm.llm_search_result_relevance import _extract_result_dois, _grade_doi_result, _normalize_doi, is_doi_query + + +def test_normalize_doi_variants(): + assert _normalize_doi("10.1016/j.ijbiomac.2025.143529") == "10.1016/j.ijbiomac.2025.143529" + assert _normalize_doi("https://doi.org/10.1038/NCOMMS7112") == "10.1038/ncomms7112" + assert _normalize_doi(":10.1111/jipb.70096") == "10.1111/jipb.70096" + assert _normalize_doi("PBPK review") == "" + assert is_doi_query("10.1016/j.ijbiomac.2025.143529") + assert not is_doi_query("PBPK review") + + +def test_extract_result_dois_from_supported_fields(): + result = { + "doi": "https://doi.org/10.1000/ABC", + "unique_id": "paper:10.2000/xyz", + "locations": [{"url": "https://doi.org/10.3000/location"}], + } + assert _extract_result_dois(result) == [ + "10.1000/abc", + "10.2000/xyz", + "10.3000/location", + ] + + +def test_extract_result_dois_ignores_non_list_locations(): + assert _extract_result_dois({"locations": True}) == [] + assert _extract_result_dois({"locations": 1}) == [] + assert _extract_result_dois({"locations": {"url": "https://doi.org/10.1000/test"}}) == [] + + +def test_doi_query_uses_exact_match(): + matched = _grade_doi_result( + "10.1016/j.ijbiomac.2025.143529", + {"doi": "https://doi.org/10.1016/j.ijbiomac.2025.143529"}, + ) + mismatched = _grade_doi_result( + "10.1016/j.ijbiomac.2025.143529", + {"doi": "https://doi.org/10.3390/plants14152362"}, + ) + + assert matched is not None and matched.score == 1.0 + assert mismatched is not None and mismatched.score == 0.0 + assert "DOI mismatch" in mismatched.reasoning + + +def test_non_doi_query_falls_back_to_llm(): + assert _grade_doi_result("PBPK相关综述", {"doi": "10.1000/test"}) is None diff --git a/test/scripts/retrieval/test_open_eval.py b/test/scripts/retrieval/test_open_eval.py index 45bd507f..dacb0973 100644 --- a/test/scripts/retrieval/test_open_eval.py +++ b/test/scripts/retrieval/test_open_eval.py @@ -82,6 +82,49 @@ def test_invalid_json(self): assert grade.error assert "JSON parse failed" in grade.error + def test_json_embedded_in_text(self): + response = 'Here is the grade:\n{"score": 0.6, "query_relevance": 0.7, "result_quality": 0.5, "content_issues": false, "confidence": 0.8, "reasoning": "ok"}' + grade = _parse_grade_response(response) + assert grade.score == 0.6 + assert grade.error == "" + + def test_lenient_parse_unescaped_quotes_in_reasoning(self): + response = ( + '{"reasoning": "The query "resilience" directly matches the result", ' + '"query_relevance": 0.95, "result_quality": 0.9, ' + '"content_issues": false, "confidence": 0.85, "score": 0.92}' + ) + grade = _parse_grade_response(response) + assert grade.error == "" + assert grade.score == 0.92 + assert grade.query_relevance == 0.95 + assert grade.result_quality == 0.9 + assert grade.content_issues is False + assert grade.confidence == 0.85 + + def test_repairs_unescaped_quotes_when_reasoning_is_last(self): + response = ( + '{"score": 0.92, "query_relevance": 0.95, "result_quality": 0.9, ' + '"content_issues": false, "confidence": 0.85, ' + '"reasoning": "The query "PBPK" directly matches"}' + ) + + grade = _parse_grade_response(response) + + assert grade.error == "" + assert grade.score == 0.92 + assert grade.reasoning == 'The query "PBPK" directly matches' + + def test_lenient_parse_when_reasoning_is_last(self): + response = '{"score": 0.7, "query_relevance": 0.8, "broken": ???, "reasoning": "final reason"}' + + grade = _parse_grade_response(response) + + assert grade.error == "" + assert grade.score == 0.7 + assert grade.query_relevance == 0.8 + assert grade.reasoning == "final reason" + def test_missing_fields_default_to_zero(self): grade = _parse_grade_response('{"score": 0.5}') assert grade.score == 0.5