From e9fff3affec0ef836821cdf98912e1a26296286e Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Tue, 21 Jul 2026 13:16:08 +0000 Subject: [PATCH 1/7] feat(benchmark): expose demographics and standings breakdown in SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add RapidataBenchmarkManager.get_demographics and get_standings_breakdown, surfacing the new leaderboard demographic endpoints so programmatic consumers can pull voter composition and per-segment standings for a benchmark. The endpoints are not in the checked-in generated OpenAPI client yet, so a thin typed wrapper (BenchmarkDemographicsApi) talks to the same low-level RapidataApiClient the generated APIs use, mirroring their request/response handling; it is superseded once the backend ships and the client is regenerated. Results are pydantic models faithful to the API contract — vote counts, shares, and the 'unknown' buckets are kept, and docstrings note that age/gender/occupation are estimated (inferred) and that segments are raw counts unless weighted scoring is requested. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: luca <25279790+LucStr@users.noreply.github.com> --- docs/mri_advanced.md | 58 ++++++++ src/rapidata/__init__.py | 5 + src/rapidata/rapidata_client/__init__.py | 7 + .../benchmark/demographics/__init__.py | 15 ++ .../demographics/_demographics_api.py | 135 ++++++++++++++++++ .../benchmark/demographics/models.py | 97 +++++++++++++ .../benchmark/rapidata_benchmark_manager.py | 117 ++++++++++++++- .../service/services/leaderboard_service.py | 15 ++ 8 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 src/rapidata/rapidata_client/benchmark/demographics/__init__.py create mode 100644 src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py create mode 100644 src/rapidata/rapidata_client/benchmark/demographics/models.py diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index 9fa72d442..c0e35eba2 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -201,6 +201,63 @@ participant = benchmark.participants[0] participant.delete() ``` +## Voter Demographics + +Every vote on a benchmark carries the voter's demographics, so you can see *who* +evaluated your models and how different groups ranked them. + +!!! note + `ageBucket`, `gender` and `occupation` are **estimated** (inferred from + behaviour), not self-declared. `country` and `language` are observed. Each + dimension includes an `"unknown"` bucket for votes whose attribute could not + be determined. + +### Voter composition + +`get_demographics` returns how the votes split across each demographic +dimension — both the raw vote count and the share of the total (shares within a +dimension sum to 1): + +```python +demographics = client.mri.get_demographics("benchmark_id_here") + +print(f"{demographics.total_votes} votes") +for bucket in demographics.dimensions["country"]: + print(f"{bucket.value}: {bucket.votes} votes ({bucket.share:.1%})") +``` + +### Standings per demographic segment + +`get_standings_breakdown` returns the overall standings plus one segment per +bucket of a dimension, each with that segment's standings — useful for spotting +where a model is ranked differently by different groups: + +```python +breakdown = client.mri.get_standings_breakdown( + "benchmark_id_here", + dimension="AgeBucket", # AgeBucket | Gender | Occupation | Country | Language +) + +for segment in breakdown.segments: + leader = segment.items[0] if segment.items else None + print(f"{segment.value} ({segment.votes} votes): {leader.name if leader else '—'}") +``` + +Segments are **raw vote counts** by default; pass `use_weighted_scoring=True` to +weight votes by annotator reliability. Both methods accept a `run_id` to scope +to a single evaluation run and a `filters` dict (the same fields as the +standings query, e.g. `tags`, `country`, `leaderboard_id`) to narrow the votes +considered: + +```python +breakdown = client.mri.get_standings_breakdown( + "benchmark_id_here", + dimension="Country", + filters={"tags": ["landscape"], "language": ["en"]}, + use_weighted_scoring=True, +) +``` + ## Accessing the Underlying Jobs The standings and win/loss matrix are aggregates. If you need the raw responses @@ -234,4 +291,5 @@ being set up) are skipped, so the list only contains jobs you can act on. - [RapidataBenchmark](/reference/rapidata/rapidata_client/benchmark/rapidata_benchmark/) - [RapidataLeaderboard](/reference/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard/) - [BenchmarkParticipant](/reference/rapidata/rapidata_client/benchmark/participant/participant/) +- [Demographics result models](/reference/rapidata/rapidata_client/benchmark/demographics/models/) diff --git a/src/rapidata/__init__.py b/src/rapidata/__init__.py index 0e55df52e..e5241ae54 100644 --- a/src/rapidata/__init__.py +++ b/src/rapidata/__init__.py @@ -20,6 +20,11 @@ RapidataValidationSet, Box, RapidataResults, + BenchmarkDemographics, + BenchmarkStandingItem, + BenchmarkStandingsBreakdown, + BenchmarkStandingsSegment, + DemographicBucket, DemographicSelection, LabelingSelection, EffortSelection, diff --git a/src/rapidata/rapidata_client/__init__.py b/src/rapidata/rapidata_client/__init__.py index 0ebd59f92..f9085ab3e 100644 --- a/src/rapidata/rapidata_client/__init__.py +++ b/src/rapidata/rapidata_client/__init__.py @@ -14,6 +14,13 @@ CostEstimate, JobProgress, ) +from .benchmark.demographics import ( + BenchmarkDemographics, + BenchmarkStandingItem, + BenchmarkStandingsBreakdown, + BenchmarkStandingsSegment, + DemographicBucket, +) from .signal import RapidataSignal, RapidataSignalManager from .validation import ValidationSetManager, RapidataValidationSet, Box from .results import RapidataResults diff --git a/src/rapidata/rapidata_client/benchmark/demographics/__init__.py b/src/rapidata/rapidata_client/benchmark/demographics/__init__.py new file mode 100644 index 000000000..bb8a43655 --- /dev/null +++ b/src/rapidata/rapidata_client/benchmark/demographics/__init__.py @@ -0,0 +1,15 @@ +from rapidata.rapidata_client.benchmark.demographics.models import ( + BenchmarkDemographics, + BenchmarkStandingItem, + BenchmarkStandingsBreakdown, + BenchmarkStandingsSegment, + DemographicBucket, +) + +__all__ = [ + "BenchmarkDemographics", + "BenchmarkStandingItem", + "BenchmarkStandingsBreakdown", + "BenchmarkStandingsSegment", + "DemographicBucket", +] diff --git a/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py b/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py new file mode 100644 index 000000000..9bec83393 --- /dev/null +++ b/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import json +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING + +from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( + AudienceAudienceIdJobsGetJobIdParameter, +) +from rapidata.rapidata_client.benchmark.demographics.models import ( + BenchmarkDemographics, + BenchmarkStandingsBreakdown, +) + +if TYPE_CHECKING: + from rapidata.rapidata_client.api.rapidata_api_client import RapidataApiClient + +# Field filters accepted by both endpoints — the same surface as +# `benchmark/{id}/standings/query`, so the backend parses them identically. +BENCHMARK_FILTER_FIELDS: Tuple[str, ...] = ( + "country", + "language", + "gender", + "age_bucket", + "occupation", + "tags", + "participant_id", + "leaderboard_id", + "run_id", + "prompt_identifier", + "voted_at", +) + +# 2xx bodies are deserialized by hand below; only the error statuses need a +# mapping so `RapidataApiClient` wraps them into `RapidataError`. +_ERROR_RESPONSE_TYPES: Dict[str, Optional[str]] = { + "400": "ValidationProblemDetails", + "401": None, + "403": None, +} + + +class BenchmarkDemographicsApi: + """Thin client for the benchmark demographics and standings-breakdown endpoints. + + These endpoints are not part of the checked-in generated OpenAPI client yet. + This wrapper talks to the same low-level ``RapidataApiClient`` the generated + APIs use and mirrors their request/response handling; once the backend ships + and the client is regenerated, the generated methods supersede it. + """ + + def __init__(self, api_client: RapidataApiClient) -> None: + self._api_client = api_client + + def get_demographics( + self, + benchmark_id: str, + filters: Optional[Dict[str, AudienceAudienceIdJobsGetJobIdParameter]] = None, + ) -> BenchmarkDemographics: + raw = self._get( + resource_path="/benchmark/{benchmarkId}/demographics", + benchmark_id=benchmark_id, + filters=filters or {}, + ) + return BenchmarkDemographics.model_validate(raw) + + def get_standings_breakdown( + self, + benchmark_id: str, + dimension: str, + filters: Optional[Dict[str, AudienceAudienceIdJobsGetJobIdParameter]] = None, + use_weighted_scoring: bool = False, + ) -> BenchmarkStandingsBreakdown: + raw = self._get( + resource_path="/benchmark/{benchmarkId}/standings/breakdown", + benchmark_id=benchmark_id, + filters=filters or {}, + extra_query=[ + ("dimension", dimension), + ("useWeightedScoring", use_weighted_scoring), + ], + ) + return BenchmarkStandingsBreakdown.model_validate(raw) + + def _get( + self, + resource_path: str, + benchmark_id: str, + filters: Dict[str, AudienceAudienceIdJobsGetJobIdParameter], + extra_query: Optional[List[Tuple[str, object]]] = None, + ) -> dict: + query_params: List[Tuple[str, object]] = [] + for field, param in filters.items(): + query_params.extend(self._explode_filter(field, param)) + if extra_query: + query_params.extend((k, v) for k, v in extra_query if v is not None) + + header_params: Dict[str, Optional[str]] = { + "Accept": self._api_client.select_header_accept(["application/json"]) + } + + _param = self._api_client.param_serialize( + method="GET", + resource_path=resource_path, + path_params={"benchmarkId": benchmark_id}, + query_params=query_params, + header_params=header_params, + auth_settings=["OpenIdConnect"], + ) + + response = self._api_client.call_api(*_param) + response.read() + api_response = self._api_client.response_deserialize( + response_data=response, + response_types_map=_ERROR_RESPONSE_TYPES, + ) + return json.loads(api_response.raw_data) + + @staticmethod + def _explode_filter( + field: str, param: AudienceAudienceIdJobsGetJobIdParameter + ) -> List[Tuple[str, object]]: + """Serialize a filter to repeated ``field[op]=value`` query params. + + Mirrors the generated ``standings/query`` serializer: list operators + (e.g. ``in``) expand into one param per value. + """ + exploded: List[Tuple[str, object]] = [] + for op, value in param.to_dict().items(): + if value is None: + continue + if isinstance(value, list): + exploded.extend((f"{field}[{op}]", item) for item in value) + else: + exploded.append((f"{field}[{op}]", value)) + return exploded diff --git a/src/rapidata/rapidata_client/benchmark/demographics/models.py b/src/rapidata/rapidata_client/benchmark/demographics/models.py new file mode 100644 index 000000000..141d9abf3 --- /dev/null +++ b/src/rapidata/rapidata_client/benchmark/demographics/models.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr + +from rapidata.api_client.models.confidence_interval import ConfidenceInterval + +# The field names of the demographic dimensions. Age, gender and occupation are +# *estimated* (inferred from behaviour), not self-declared by the annotator; +# country and language are observed. Kept faithful to what the API returns. +DemographicDimension = str + + +class DemographicBucket(BaseModel): + """One value of a demographic dimension and its share of the votes. + + `value` is the bucket label (e.g. "18-24", "US", "Male"). The literal + `"unknown"` bucket carries the votes whose attribute could not be + determined; it is always present and never dropped. + """ + + model_config = ConfigDict(populate_by_name=True) + + value: StrictStr + votes: StrictInt + # Fraction of the dimension's votes in [0, 1]; the shares of a dimension + # (including "unknown") sum to 1. + share: Union[StrictFloat, StrictInt] + + +class BenchmarkDemographics(BaseModel): + """Voter composition of a benchmark, broken down by demographic dimension. + + `dimensions` maps a dimension name (`ageBucket`, `gender`, `occupation`, + `country`, `language`) to its buckets. Each dimension includes an + `"unknown"` bucket and its shares sum to 1. Age, gender and occupation are + estimated (inferred), not self-declared. + """ + + model_config = ConfigDict(populate_by_name=True) + + total_votes: StrictInt = Field(alias="totalVotes") + dimensions: Dict[DemographicDimension, List[DemographicBucket]] + + +class BenchmarkStandingItem(BaseModel): + """A single model's standing within a benchmark. + + `score`/`wins`/`total_matches` are raw vote counts unless the breakdown was + requested with weighted scoring, in which case they are reliability-weighted. + """ + + model_config = ConfigDict(populate_by_name=True) + + id: StrictStr + name: StrictStr + family: Optional[StrictStr] = None + score: Optional[Union[StrictFloat, StrictInt]] = None + wins: Union[StrictFloat, StrictInt] + total_matches: Union[StrictFloat, StrictInt] = Field(alias="totalMatches") + confidence_interval: Optional[ConfidenceInterval] = Field( + default=None, alias="confidenceInterval" + ) + + +class BenchmarkStandingsSegment(BaseModel): + """Standings restricted to voters in one bucket of a demographic dimension. + + `value` is the bucket label (including `"unknown"`), `votes` is the raw + number of votes contributed by that segment, and `items` are that segment's + standings. + """ + + model_config = ConfigDict(populate_by_name=True) + + value: StrictStr + votes: StrictInt + items: List[BenchmarkStandingItem] + + +class BenchmarkStandingsBreakdown(BaseModel): + """Per-segment standings for one demographic dimension of a benchmark. + + `global_standings` is the overall standings across all voters; `segments` + holds one entry per bucket of `dimension` (including `"unknown"`). Segments + are raw vote counts unless weighted scoring was requested. Age, gender and + occupation dimensions are estimated (inferred), not self-declared. + """ + + model_config = ConfigDict(populate_by_name=True) + + dimension: StrictStr + # `global` is a Python keyword, so the attribute is exposed as + # `global_standings` while still (de)serializing to the API's `global` key. + global_standings: List[BenchmarkStandingItem] = Field(alias="global") + segments: List[BenchmarkStandingsSegment] diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py index 750bd5760..8bed1ef24 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py @@ -1,5 +1,14 @@ -from typing import Optional +from __future__ import annotations + +from typing import Literal, Optional, Union from rapidata.rapidata_client.benchmark.rapidata_benchmark import RapidataBenchmark +from rapidata.rapidata_client.benchmark.demographics._demographics_api import ( + BENCHMARK_FILTER_FIELDS, +) +from rapidata.rapidata_client.benchmark.demographics.models import ( + BenchmarkDemographics, + BenchmarkStandingsBreakdown, +) from rapidata.api_client.models.create_benchmark_endpoint_input import ( CreateBenchmarkEndpointInput, ) @@ -9,6 +18,10 @@ from rapidata.service.openapi_service import OpenAPIService from rapidata.rapidata_client.config import logger, tracer +# The demographic dimension to break standings down by, as accepted by the +# breakdown endpoint's `dimension` query parameter. +BreakdownDimension = Literal["AgeBucket", "Gender", "Occupation", "Country", "Language"] + class RapidataBenchmarkManager: """ @@ -121,3 +134,105 @@ def find_benchmarks( RapidataBenchmark(benchmark.name, benchmark.id, self.__openapi_service) for benchmark in benchmark_result.items ] + + def get_demographics( + self, + benchmark_id: str, + run_id: Optional[str] = None, + filters: Optional[dict[str, Union[str, list[str]]]] = None, + ) -> BenchmarkDemographics: + """ + Returns the demographic composition of the voters for a benchmark. + + The result reports, per dimension (``ageBucket``, ``gender``, + ``occupation``, ``country``, ``language``), how the votes split across + that dimension's buckets — both the raw vote count and the share of the + total. Each dimension includes an ``"unknown"`` bucket for votes whose + attribute could not be determined, and the shares of a dimension sum to 1. + + Note that ``ageBucket``, ``gender`` and ``occupation`` are **estimated** + (inferred from behaviour), not self-declared by the annotators; + ``country`` and ``language`` are observed. + + Args: + benchmark_id: The id of the benchmark. + run_id: Restrict the composition to a single evaluation run. If None, all runs are considered. + filters: Additional filters on the underlying votes, keyed by field name. Accepts the same fields as the standings query (e.g. ``country``, ``language``, ``gender``, ``age_bucket``, ``occupation``, ``tags``, ``leaderboard_id``, ``participant_id``, ``prompt_identifier``). Each value is a string or list of strings matched with ``in`` semantics. + + Returns: + A BenchmarkDemographics describing the voter composition. + """ + with tracer.start_as_current_span("RapidataBenchmarkManager.get_demographics"): + return self.__openapi_service.leaderboard.benchmark_demographics_api.get_demographics( + benchmark_id=benchmark_id, + filters=self.__build_filters(filters, run_id), + ) + + def get_standings_breakdown( + self, + benchmark_id: str, + dimension: BreakdownDimension, + run_id: Optional[str] = None, + filters: Optional[dict[str, Union[str, list[str]]]] = None, + use_weighted_scoring: bool = False, + ) -> BenchmarkStandingsBreakdown: + """ + Returns the benchmark standings broken down by a demographic dimension. + + Alongside the global standings, this returns one segment per bucket of + ``dimension`` (including an ``"unknown"`` bucket), each with that + segment's raw vote count and its own standings. This shows how different + voter groups rank the models relative to each other. + + By default segments are **raw vote counts**; pass + ``use_weighted_scoring=True`` to weight votes by annotator reliability. + Note that the ``AgeBucket``, ``Gender`` and ``Occupation`` dimensions are + **estimated** (inferred), not self-declared. + + Args: + benchmark_id: The id of the benchmark. + dimension: The demographic dimension to break the standings down by. One of "AgeBucket", "Gender", "Occupation", "Country", "Language". + run_id: Restrict the breakdown to a single evaluation run. If None, all runs are considered. + filters: Additional filters on the underlying votes, keyed by field name (same fields as :py:meth:`get_demographics`). Each value is a string or list of strings matched with ``in`` semantics. + use_weighted_scoring: Whether to weight votes by annotator reliability instead of using raw counts. Defaults to False. + + Returns: + A BenchmarkStandingsBreakdown with the global standings and per-segment standings. + """ + with tracer.start_as_current_span( + "RapidataBenchmarkManager.get_standings_breakdown" + ): + if dimension not in BreakdownDimension.__args__: + raise ValueError( + "Dimension must be one of: " + + ", ".join(BreakdownDimension.__args__) + ) + + return self.__openapi_service.leaderboard.benchmark_demographics_api.get_standings_breakdown( + benchmark_id=benchmark_id, + dimension=dimension, + filters=self.__build_filters(filters, run_id), + use_weighted_scoring=use_weighted_scoring, + ) + + @staticmethod + def __build_filters( + filters: Optional[dict[str, Union[str, list[str]]]], + run_id: Optional[str], + ) -> dict[str, AudienceAudienceIdJobsGetJobIdParameter]: + raw: dict[str, Union[str, list[str]]] = dict(filters or {}) + if run_id is not None: + raw["run_id"] = run_id + + built: dict[str, AudienceAudienceIdJobsGetJobIdParameter] = {} + for field, value in raw.items(): + if field not in BENCHMARK_FILTER_FIELDS: + raise ValueError( + f"Unknown filter field '{field}'. Allowed fields: " + + ", ".join(BENCHMARK_FILTER_FIELDS) + ) + values = [value] if isinstance(value, str) else list(value) + param = AudienceAudienceIdJobsGetJobIdParameter() + param.var_in = values + built[field] = param + return built diff --git a/src/rapidata/service/services/leaderboard_service.py b/src/rapidata/service/services/leaderboard_service.py index d0e692e07..1aa737aad 100644 --- a/src/rapidata/service/services/leaderboard_service.py +++ b/src/rapidata/service/services/leaderboard_service.py @@ -7,6 +7,9 @@ from rapidata.api_client.api.benchmark_api import BenchmarkApi from rapidata.api_client.api.participant_api import ParticipantApi from rapidata.rapidata_client.api.rapidata_api_client import RapidataApiClient + from rapidata.rapidata_client.benchmark.demographics._demographics_api import ( + BenchmarkDemographicsApi, + ) class LeaderboardService: @@ -15,6 +18,7 @@ def __init__(self, api_client: RapidataApiClient) -> None: self._leaderboard_api: LeaderboardApi | None = None self._benchmark_api: BenchmarkApi | None = None self._participant_api: ParticipantApi | None = None + self._benchmark_demographics_api: BenchmarkDemographicsApi | None = None @property def leaderboard_api(self) -> LeaderboardApi: @@ -36,3 +40,14 @@ def participant_api(self) -> ParticipantApi: from rapidata.api_client.api.participant_api import ParticipantApi self._participant_api = ParticipantApi(self._api_client) return self._participant_api + + @property + def benchmark_demographics_api(self) -> BenchmarkDemographicsApi: + # Hand-written wrapper for endpoints not yet in the generated client; see + # rapidata_client/benchmark/demographics/_demographics_api.py. + if self._benchmark_demographics_api is None: + from rapidata.rapidata_client.benchmark.demographics._demographics_api import ( + BenchmarkDemographicsApi, + ) + self._benchmark_demographics_api = BenchmarkDemographicsApi(self._api_client) + return self._benchmark_demographics_api From 9e1bdcc8d646d7c6deeaf500895986964842e58b Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Wed, 22 Jul 2026 13:52:00 +0000 Subject: [PATCH 2/7] fix(benchmark): drop weighting param and complete standings item shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconcile with the merged backend leg (rapidata-backend#4821), which locked both new endpoints as raw-count only: - Remove use_weighted_scoring from get_standings_breakdown and the thin wrapper — neither /demographics nor /standings/breakdown exposes a weighting param (the pre-existing standings query still does). - Model the full locked standings item shape: add proprietaryName, logo, status (StandingStatus) and isDisabled alongside the existing fields. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: luca <25279790+LucStr@users.noreply.github.com> --- docs/mri_advanced.md | 9 +++----- .../demographics/_demographics_api.py | 6 +---- .../benchmark/demographics/models.py | 23 +++++++++++++++---- .../benchmark/rapidata_benchmark_manager.py | 10 +++----- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index c0e35eba2..5c563ff95 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -243,18 +243,15 @@ for segment in breakdown.segments: print(f"{segment.value} ({segment.votes} votes): {leader.name if leader else '—'}") ``` -Segments are **raw vote counts** by default; pass `use_weighted_scoring=True` to -weight votes by annotator reliability. Both methods accept a `run_id` to scope -to a single evaluation run and a `filters` dict (the same fields as the -standings query, e.g. `tags`, `country`, `leaderboard_id`) to narrow the votes -considered: +Segments are **raw vote counts**. Both methods accept a `run_id` to scope to a +single evaluation run and a `filters` dict (the same fields as the standings +query, e.g. `tags`, `country`, `leaderboard_id`) to narrow the votes considered: ```python breakdown = client.mri.get_standings_breakdown( "benchmark_id_here", dimension="Country", filters={"tags": ["landscape"], "language": ["en"]}, - use_weighted_scoring=True, ) ``` diff --git a/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py b/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py index 9bec83393..83463c792 100644 --- a/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py +++ b/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py @@ -68,16 +68,12 @@ def get_standings_breakdown( benchmark_id: str, dimension: str, filters: Optional[Dict[str, AudienceAudienceIdJobsGetJobIdParameter]] = None, - use_weighted_scoring: bool = False, ) -> BenchmarkStandingsBreakdown: raw = self._get( resource_path="/benchmark/{benchmarkId}/standings/breakdown", benchmark_id=benchmark_id, filters=filters or {}, - extra_query=[ - ("dimension", dimension), - ("useWeightedScoring", use_weighted_scoring), - ], + extra_query=[("dimension", dimension)], ) return BenchmarkStandingsBreakdown.model_validate(raw) diff --git a/src/rapidata/rapidata_client/benchmark/demographics/models.py b/src/rapidata/rapidata_client/benchmark/demographics/models.py index 141d9abf3..f3c881fb5 100644 --- a/src/rapidata/rapidata_client/benchmark/demographics/models.py +++ b/src/rapidata/rapidata_client/benchmark/demographics/models.py @@ -2,9 +2,18 @@ from typing import Dict, List, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, +) from rapidata.api_client.models.confidence_interval import ConfidenceInterval +from rapidata.api_client.models.standing_status import StandingStatus # The field names of the demographic dimensions. Age, gender and occupation are # *estimated* (inferred from behaviour), not self-declared by the annotator; @@ -47,8 +56,8 @@ class BenchmarkDemographics(BaseModel): class BenchmarkStandingItem(BaseModel): """A single model's standing within a benchmark. - `score`/`wins`/`total_matches` are raw vote counts unless the breakdown was - requested with weighted scoring, in which case they are reliability-weighted. + `score`/`wins`/`total_matches` are raw vote counts. Mirrors the standings + item shape returned by the benchmark standings endpoints. """ model_config = ConfigDict(populate_by_name=True) @@ -56,9 +65,13 @@ class BenchmarkStandingItem(BaseModel): id: StrictStr name: StrictStr family: Optional[StrictStr] = None + proprietary_name: Optional[StrictStr] = Field(default=None, alias="proprietaryName") + logo: Optional[StrictStr] = None + status: StandingStatus score: Optional[Union[StrictFloat, StrictInt]] = None wins: Union[StrictFloat, StrictInt] total_matches: Union[StrictFloat, StrictInt] = Field(alias="totalMatches") + is_disabled: StrictBool = Field(alias="isDisabled") confidence_interval: Optional[ConfidenceInterval] = Field( default=None, alias="confidenceInterval" ) @@ -84,8 +97,8 @@ class BenchmarkStandingsBreakdown(BaseModel): `global_standings` is the overall standings across all voters; `segments` holds one entry per bucket of `dimension` (including `"unknown"`). Segments - are raw vote counts unless weighted scoring was requested. Age, gender and - occupation dimensions are estimated (inferred), not self-declared. + are raw vote counts. Age, gender and occupation dimensions are estimated + (inferred), not self-declared. """ model_config = ConfigDict(populate_by_name=True) diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py index 8bed1ef24..70b945c73 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py @@ -174,7 +174,6 @@ def get_standings_breakdown( dimension: BreakdownDimension, run_id: Optional[str] = None, filters: Optional[dict[str, Union[str, list[str]]]] = None, - use_weighted_scoring: bool = False, ) -> BenchmarkStandingsBreakdown: """ Returns the benchmark standings broken down by a demographic dimension. @@ -184,17 +183,15 @@ def get_standings_breakdown( segment's raw vote count and its own standings. This shows how different voter groups rank the models relative to each other. - By default segments are **raw vote counts**; pass - ``use_weighted_scoring=True`` to weight votes by annotator reliability. - Note that the ``AgeBucket``, ``Gender`` and ``Occupation`` dimensions are - **estimated** (inferred), not self-declared. + Segments are **raw vote counts**. Note that the ``AgeBucket``, ``Gender`` + and ``Occupation`` dimensions are **estimated** (inferred), not + self-declared. Args: benchmark_id: The id of the benchmark. dimension: The demographic dimension to break the standings down by. One of "AgeBucket", "Gender", "Occupation", "Country", "Language". run_id: Restrict the breakdown to a single evaluation run. If None, all runs are considered. filters: Additional filters on the underlying votes, keyed by field name (same fields as :py:meth:`get_demographics`). Each value is a string or list of strings matched with ``in`` semantics. - use_weighted_scoring: Whether to weight votes by annotator reliability instead of using raw counts. Defaults to False. Returns: A BenchmarkStandingsBreakdown with the global standings and per-segment standings. @@ -212,7 +209,6 @@ def get_standings_breakdown( benchmark_id=benchmark_id, dimension=dimension, filters=self.__build_filters(filters, run_id), - use_weighted_scoring=use_weighted_scoring, ) @staticmethod From 17734b5782e173ca776cc06cb2718192c2bf441a Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Wed, 22 Jul 2026 13:59:56 +0000 Subject: [PATCH 3/7] ci: re-trigger checks Co-Authored-By: Claude Opus 4.8 Co-Authored-By: luca <25279790+LucStr@users.noreply.github.com> From 9e2374a46d3e5d176ea3244021be829a48c2b924 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Thu, 23 Jul 2026 13:45:18 +0000 Subject: [PATCH 4/7] refactor(benchmark): move demographics onto the benchmark object as DataFrames Address review feedback (@LinoGiger): expose the demographic data the way the rest of the benchmark API is exposed instead of as manager methods returning bespoke pydantic models. - Move get_demographics / get_standings_breakdown onto RapidataBenchmark (called as benchmark.get_demographics(), like get_overall_standings), dropping the benchmark_id argument. - Return pandas DataFrames matching the existing standings methods, with explicit tags / leaderboard_ids / run_id filters instead of a filters dict. Vote counts, shares and the 'unknown' buckets are kept as columns/rows; docstrings note the estimated (inferred) dimensions. - get_standings_breakdown returns only the per-segment standings; the overall standings already live on get_overall_standings. - Drop the five public pydantic result classes and their re-exports; the thin wrapper now returns the parsed JSON body. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: luca <25279790+LucStr@users.noreply.github.com> --- docs/mri_advanced.md | 42 +++-- src/rapidata/__init__.py | 5 - src/rapidata/rapidata_client/__init__.py | 7 - .../benchmark/demographics/__init__.py | 15 -- .../demographics/_demographics_api.py | 37 +---- .../benchmark/demographics/models.py | 110 -------------- .../benchmark/rapidata_benchmark.py | 143 ++++++++++++++++++ .../benchmark/rapidata_benchmark_manager.py | 113 +------------- 8 files changed, 170 insertions(+), 302 deletions(-) delete mode 100644 src/rapidata/rapidata_client/benchmark/demographics/models.py diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index 5c563ff95..11e2398c0 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -204,7 +204,8 @@ participant.delete() ## Voter Demographics Every vote on a benchmark carries the voter's demographics, so you can see *who* -evaluated your models and how different groups ranked them. +evaluated your models and how different groups ranked them. Both methods live on +the benchmark and return a pandas DataFrame, like `get_overall_standings`. !!! note `ageBucket`, `gender` and `occupation` are **estimated** (inferred from @@ -214,44 +215,38 @@ evaluated your models and how different groups ranked them. ### Voter composition -`get_demographics` returns how the votes split across each demographic -dimension — both the raw vote count and the share of the total (shares within a -dimension sum to 1): +`get_demographics` returns one row per `(dimension, value)` with the raw vote +count and its share of the dimension (shares within a dimension sum to 1): ```python -demographics = client.mri.get_demographics("benchmark_id_here") +demographics = benchmark.get_demographics() +# columns: dimension, value, votes, share -print(f"{demographics.total_votes} votes") -for bucket in demographics.dimensions["country"]: - print(f"{bucket.value}: {bucket.votes} votes ({bucket.share:.1%})") +countries = demographics[demographics["dimension"] == "country"] +print(countries) ``` ### Standings per demographic segment -`get_standings_breakdown` returns the overall standings plus one segment per -bucket of a dimension, each with that segment's standings — useful for spotting -where a model is ranked differently by different groups: +`get_standings_breakdown` returns one row per `(segment, model)` — how each +demographic segment of voters ranks the models, with that segment's vote count. +Useful for spotting where a model is ranked differently by different groups. For +the overall standings across all voters, use `get_overall_standings`. ```python -breakdown = client.mri.get_standings_breakdown( - "benchmark_id_here", +breakdown = benchmark.get_standings_breakdown( dimension="AgeBucket", # AgeBucket | Gender | Occupation | Country | Language ) - -for segment in breakdown.segments: - leader = segment.items[0] if segment.items else None - print(f"{segment.value} ({segment.votes} votes): {leader.name if leader else '—'}") +# columns: segment, segment_votes, name, wins, total_matches, score ``` -Segments are **raw vote counts**. Both methods accept a `run_id` to scope to a -single evaluation run and a `filters` dict (the same fields as the standings -query, e.g. `tags`, `country`, `leaderboard_id`) to narrow the votes considered: +Segments are **raw vote counts**. Both methods accept `tags`, `leaderboard_ids` +and `run_id` to narrow the votes considered: ```python -breakdown = client.mri.get_standings_breakdown( - "benchmark_id_here", +breakdown = benchmark.get_standings_breakdown( dimension="Country", - filters={"tags": ["landscape"], "language": ["en"]}, + tags=["landscape"], ) ``` @@ -288,5 +283,4 @@ being set up) are skipped, so the list only contains jobs you can act on. - [RapidataBenchmark](/reference/rapidata/rapidata_client/benchmark/rapidata_benchmark/) - [RapidataLeaderboard](/reference/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard/) - [BenchmarkParticipant](/reference/rapidata/rapidata_client/benchmark/participant/participant/) -- [Demographics result models](/reference/rapidata/rapidata_client/benchmark/demographics/models/) diff --git a/src/rapidata/__init__.py b/src/rapidata/__init__.py index e5241ae54..0e55df52e 100644 --- a/src/rapidata/__init__.py +++ b/src/rapidata/__init__.py @@ -20,11 +20,6 @@ RapidataValidationSet, Box, RapidataResults, - BenchmarkDemographics, - BenchmarkStandingItem, - BenchmarkStandingsBreakdown, - BenchmarkStandingsSegment, - DemographicBucket, DemographicSelection, LabelingSelection, EffortSelection, diff --git a/src/rapidata/rapidata_client/__init__.py b/src/rapidata/rapidata_client/__init__.py index f9085ab3e..0ebd59f92 100644 --- a/src/rapidata/rapidata_client/__init__.py +++ b/src/rapidata/rapidata_client/__init__.py @@ -14,13 +14,6 @@ CostEstimate, JobProgress, ) -from .benchmark.demographics import ( - BenchmarkDemographics, - BenchmarkStandingItem, - BenchmarkStandingsBreakdown, - BenchmarkStandingsSegment, - DemographicBucket, -) from .signal import RapidataSignal, RapidataSignalManager from .validation import ValidationSetManager, RapidataValidationSet, Box from .results import RapidataResults diff --git a/src/rapidata/rapidata_client/benchmark/demographics/__init__.py b/src/rapidata/rapidata_client/benchmark/demographics/__init__.py index bb8a43655..e69de29bb 100644 --- a/src/rapidata/rapidata_client/benchmark/demographics/__init__.py +++ b/src/rapidata/rapidata_client/benchmark/demographics/__init__.py @@ -1,15 +0,0 @@ -from rapidata.rapidata_client.benchmark.demographics.models import ( - BenchmarkDemographics, - BenchmarkStandingItem, - BenchmarkStandingsBreakdown, - BenchmarkStandingsSegment, - DemographicBucket, -) - -__all__ = [ - "BenchmarkDemographics", - "BenchmarkStandingItem", - "BenchmarkStandingsBreakdown", - "BenchmarkStandingsSegment", - "DemographicBucket", -] diff --git a/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py b/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py index 83463c792..e2147dc17 100644 --- a/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py +++ b/src/rapidata/rapidata_client/benchmark/demographics/_demographics_api.py @@ -6,31 +6,11 @@ from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( AudienceAudienceIdJobsGetJobIdParameter, ) -from rapidata.rapidata_client.benchmark.demographics.models import ( - BenchmarkDemographics, - BenchmarkStandingsBreakdown, -) if TYPE_CHECKING: from rapidata.rapidata_client.api.rapidata_api_client import RapidataApiClient -# Field filters accepted by both endpoints — the same surface as -# `benchmark/{id}/standings/query`, so the backend parses them identically. -BENCHMARK_FILTER_FIELDS: Tuple[str, ...] = ( - "country", - "language", - "gender", - "age_bucket", - "occupation", - "tags", - "participant_id", - "leaderboard_id", - "run_id", - "prompt_identifier", - "voted_at", -) - -# 2xx bodies are deserialized by hand below; only the error statuses need a +# 2xx bodies are returned as parsed dicts; only the error statuses need a # mapping so `RapidataApiClient` wraps them into `RapidataError`. _ERROR_RESPONSE_TYPES: Dict[str, Optional[str]] = { "400": "ValidationProblemDetails", @@ -44,8 +24,9 @@ class BenchmarkDemographicsApi: These endpoints are not part of the checked-in generated OpenAPI client yet. This wrapper talks to the same low-level ``RapidataApiClient`` the generated - APIs use and mirrors their request/response handling; once the backend ships - and the client is regenerated, the generated methods supersede it. + APIs use and mirrors their request/response handling, returning the parsed + JSON body; once the backend ships and the client is regenerated, the + generated methods supersede it. """ def __init__(self, api_client: RapidataApiClient) -> None: @@ -55,27 +36,25 @@ def get_demographics( self, benchmark_id: str, filters: Optional[Dict[str, AudienceAudienceIdJobsGetJobIdParameter]] = None, - ) -> BenchmarkDemographics: - raw = self._get( + ) -> dict: + return self._get( resource_path="/benchmark/{benchmarkId}/demographics", benchmark_id=benchmark_id, filters=filters or {}, ) - return BenchmarkDemographics.model_validate(raw) def get_standings_breakdown( self, benchmark_id: str, dimension: str, filters: Optional[Dict[str, AudienceAudienceIdJobsGetJobIdParameter]] = None, - ) -> BenchmarkStandingsBreakdown: - raw = self._get( + ) -> dict: + return self._get( resource_path="/benchmark/{benchmarkId}/standings/breakdown", benchmark_id=benchmark_id, filters=filters or {}, extra_query=[("dimension", dimension)], ) - return BenchmarkStandingsBreakdown.model_validate(raw) def _get( self, diff --git a/src/rapidata/rapidata_client/benchmark/demographics/models.py b/src/rapidata/rapidata_client/benchmark/demographics/models.py deleted file mode 100644 index f3c881fb5..000000000 --- a/src/rapidata/rapidata_client/benchmark/demographics/models.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -from typing import Dict, List, Optional, Union - -from pydantic import ( - BaseModel, - ConfigDict, - Field, - StrictBool, - StrictFloat, - StrictInt, - StrictStr, -) - -from rapidata.api_client.models.confidence_interval import ConfidenceInterval -from rapidata.api_client.models.standing_status import StandingStatus - -# The field names of the demographic dimensions. Age, gender and occupation are -# *estimated* (inferred from behaviour), not self-declared by the annotator; -# country and language are observed. Kept faithful to what the API returns. -DemographicDimension = str - - -class DemographicBucket(BaseModel): - """One value of a demographic dimension and its share of the votes. - - `value` is the bucket label (e.g. "18-24", "US", "Male"). The literal - `"unknown"` bucket carries the votes whose attribute could not be - determined; it is always present and never dropped. - """ - - model_config = ConfigDict(populate_by_name=True) - - value: StrictStr - votes: StrictInt - # Fraction of the dimension's votes in [0, 1]; the shares of a dimension - # (including "unknown") sum to 1. - share: Union[StrictFloat, StrictInt] - - -class BenchmarkDemographics(BaseModel): - """Voter composition of a benchmark, broken down by demographic dimension. - - `dimensions` maps a dimension name (`ageBucket`, `gender`, `occupation`, - `country`, `language`) to its buckets. Each dimension includes an - `"unknown"` bucket and its shares sum to 1. Age, gender and occupation are - estimated (inferred), not self-declared. - """ - - model_config = ConfigDict(populate_by_name=True) - - total_votes: StrictInt = Field(alias="totalVotes") - dimensions: Dict[DemographicDimension, List[DemographicBucket]] - - -class BenchmarkStandingItem(BaseModel): - """A single model's standing within a benchmark. - - `score`/`wins`/`total_matches` are raw vote counts. Mirrors the standings - item shape returned by the benchmark standings endpoints. - """ - - model_config = ConfigDict(populate_by_name=True) - - id: StrictStr - name: StrictStr - family: Optional[StrictStr] = None - proprietary_name: Optional[StrictStr] = Field(default=None, alias="proprietaryName") - logo: Optional[StrictStr] = None - status: StandingStatus - score: Optional[Union[StrictFloat, StrictInt]] = None - wins: Union[StrictFloat, StrictInt] - total_matches: Union[StrictFloat, StrictInt] = Field(alias="totalMatches") - is_disabled: StrictBool = Field(alias="isDisabled") - confidence_interval: Optional[ConfidenceInterval] = Field( - default=None, alias="confidenceInterval" - ) - - -class BenchmarkStandingsSegment(BaseModel): - """Standings restricted to voters in one bucket of a demographic dimension. - - `value` is the bucket label (including `"unknown"`), `votes` is the raw - number of votes contributed by that segment, and `items` are that segment's - standings. - """ - - model_config = ConfigDict(populate_by_name=True) - - value: StrictStr - votes: StrictInt - items: List[BenchmarkStandingItem] - - -class BenchmarkStandingsBreakdown(BaseModel): - """Per-segment standings for one demographic dimension of a benchmark. - - `global_standings` is the overall standings across all voters; `segments` - holds one entry per bucket of `dimension` (including `"unknown"`). Segments - are raw vote counts. Age, gender and occupation dimensions are estimated - (inferred), not self-declared. - """ - - model_config = ConfigDict(populate_by_name=True) - - dimension: StrictStr - # `global` is a Python keyword, so the attribute is exposed as - # `global_standings` while still (de)serializing to the API's `global` key. - global_standings: List[BenchmarkStandingItem] = Field(alias="global") - segments: List[BenchmarkStandingsSegment] diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 45e4e66db..ecc7b4642 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -27,6 +27,11 @@ from rapidata.rapidata_client.settings import RapidataSetting from rapidata.service.openapi_service import OpenAPIService +# The demographic dimension to split standings by, as accepted by the +# breakdown endpoint. Age, gender and occupation are estimated (inferred); +# country and language are observed. +BreakdownDimension = Literal["AgeBucket", "Gender", "Occupation", "Country", "Language"] + class RapidataBenchmark: """ @@ -806,6 +811,144 @@ def get_win_loss_matrix( columns=pd.Index(result.columns), ) + def get_demographics( + self, + tags: Optional[list[str]] = None, + leaderboard_ids: Optional[list[str]] = None, + run_id: Optional[str] = None, + ) -> pd.DataFrame: + """ + Returns the demographic composition of the voters for this benchmark. + + One row per (dimension, bucket): the ``votes`` cast by that bucket and + its ``share`` of the dimension's votes (a dimension's shares sum to 1). + Every dimension (``ageBucket``, ``gender``, ``occupation``, ``country``, + ``language``) includes an ``"unknown"`` bucket for votes whose attribute + could not be determined. + + ``ageBucket``, ``gender`` and ``occupation`` are estimated (inferred from + behaviour), not self-declared; ``country`` and ``language`` are observed. + + Args: + tags: Only count votes on matchups with these tags. If None, all matchups are considered. + leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered. + run_id: Only count votes from this evaluation run. If None, all runs are considered. + + Returns: + A pandas DataFrame with columns ``dimension``, ``value``, ``votes``, ``share``. + """ + import pandas as pd + + with tracer.start_as_current_span("RapidataBenchmark.get_demographics"): + data = self._openapi_service.leaderboard.benchmark_demographics_api.get_demographics( + benchmark_id=self.id, + filters=self.__demographic_filters(tags, leaderboard_ids, run_id), + ) + + rows = [ + { + "dimension": dimension, + "value": bucket["value"], + "votes": bucket["votes"], + "share": bucket["share"], + } + for dimension, buckets in data["dimensions"].items() + for bucket in buckets + ] + + return pd.DataFrame( + rows, columns=pd.Index(["dimension", "value", "votes", "share"]) + ) + + def get_standings_breakdown( + self, + dimension: BreakdownDimension, + tags: Optional[list[str]] = None, + leaderboard_ids: Optional[list[str]] = None, + run_id: Optional[str] = None, + ) -> pd.DataFrame: + """ + Returns the standings split by a demographic dimension of the voters. + + One row per (segment, model): how each demographic segment of voters + ranks the models, alongside that segment's vote count. The segments + include an ``"unknown"`` bucket. Scores are raw vote counts. For the + overall standings across all voters, use :py:meth:`get_overall_standings`. + + ``AgeBucket``, ``Gender`` and ``Occupation`` are estimated (inferred), not + self-declared; ``Country`` and ``Language`` are observed. + + Args: + dimension: The demographic dimension to split by. One of "AgeBucket", "Gender", "Occupation", "Country", "Language". + tags: Only count votes on matchups with these tags. If None, all matchups are considered. + leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered. + run_id: Only count votes from this evaluation run. If None, all runs are considered. + + Returns: + A pandas DataFrame with columns ``segment``, ``segment_votes``, ``name``, ``wins``, ``total_matches``, ``score``. + """ + import pandas as pd + + with tracer.start_as_current_span("RapidataBenchmark.get_standings_breakdown"): + if dimension not in BreakdownDimension.__args__: + raise ValueError( + "Dimension must be one of: " + + ", ".join(BreakdownDimension.__args__) + ) + + data = self._openapi_service.leaderboard.benchmark_demographics_api.get_standings_breakdown( + benchmark_id=self.id, + dimension=dimension, + filters=self.__demographic_filters(tags, leaderboard_ids, run_id), + ) + + rows = [] + for segment in data["segments"]: + for item in segment["items"]: + score = item.get("score") + rows.append( + { + "segment": segment["value"], + "segment_votes": segment["votes"], + "name": item["name"], + "wins": item["wins"], + "total_matches": item["totalMatches"], + "score": round(score, 2) if score is not None else None, + } + ) + + return pd.DataFrame( + rows, + columns=pd.Index( + [ + "segment", + "segment_votes", + "name", + "wins", + "total_matches", + "score", + ] + ), + ) + + def __demographic_filters( + self, + tags: Optional[list[str]], + leaderboard_ids: Optional[list[str]], + run_id: Optional[str], + ) -> dict[str, AudienceAudienceIdJobsGetJobIdParameter]: + filters: dict[str, AudienceAudienceIdJobsGetJobIdParameter] = {} + for field, values in ( + ("tags", tags), + ("leaderboard_id", leaderboard_ids), + ("run_id", [run_id] if run_id is not None else None), + ): + if values: + param = AudienceAudienceIdJobsGetJobIdParameter() + param.var_in = values + filters[field] = param + return filters + def __str__(self) -> str: return f"RapidataBenchmark(name={self.name}, id={self.id})" diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py index 70b945c73..750bd5760 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py @@ -1,14 +1,5 @@ -from __future__ import annotations - -from typing import Literal, Optional, Union +from typing import Optional from rapidata.rapidata_client.benchmark.rapidata_benchmark import RapidataBenchmark -from rapidata.rapidata_client.benchmark.demographics._demographics_api import ( - BENCHMARK_FILTER_FIELDS, -) -from rapidata.rapidata_client.benchmark.demographics.models import ( - BenchmarkDemographics, - BenchmarkStandingsBreakdown, -) from rapidata.api_client.models.create_benchmark_endpoint_input import ( CreateBenchmarkEndpointInput, ) @@ -18,10 +9,6 @@ from rapidata.service.openapi_service import OpenAPIService from rapidata.rapidata_client.config import logger, tracer -# The demographic dimension to break standings down by, as accepted by the -# breakdown endpoint's `dimension` query parameter. -BreakdownDimension = Literal["AgeBucket", "Gender", "Occupation", "Country", "Language"] - class RapidataBenchmarkManager: """ @@ -134,101 +121,3 @@ def find_benchmarks( RapidataBenchmark(benchmark.name, benchmark.id, self.__openapi_service) for benchmark in benchmark_result.items ] - - def get_demographics( - self, - benchmark_id: str, - run_id: Optional[str] = None, - filters: Optional[dict[str, Union[str, list[str]]]] = None, - ) -> BenchmarkDemographics: - """ - Returns the demographic composition of the voters for a benchmark. - - The result reports, per dimension (``ageBucket``, ``gender``, - ``occupation``, ``country``, ``language``), how the votes split across - that dimension's buckets — both the raw vote count and the share of the - total. Each dimension includes an ``"unknown"`` bucket for votes whose - attribute could not be determined, and the shares of a dimension sum to 1. - - Note that ``ageBucket``, ``gender`` and ``occupation`` are **estimated** - (inferred from behaviour), not self-declared by the annotators; - ``country`` and ``language`` are observed. - - Args: - benchmark_id: The id of the benchmark. - run_id: Restrict the composition to a single evaluation run. If None, all runs are considered. - filters: Additional filters on the underlying votes, keyed by field name. Accepts the same fields as the standings query (e.g. ``country``, ``language``, ``gender``, ``age_bucket``, ``occupation``, ``tags``, ``leaderboard_id``, ``participant_id``, ``prompt_identifier``). Each value is a string or list of strings matched with ``in`` semantics. - - Returns: - A BenchmarkDemographics describing the voter composition. - """ - with tracer.start_as_current_span("RapidataBenchmarkManager.get_demographics"): - return self.__openapi_service.leaderboard.benchmark_demographics_api.get_demographics( - benchmark_id=benchmark_id, - filters=self.__build_filters(filters, run_id), - ) - - def get_standings_breakdown( - self, - benchmark_id: str, - dimension: BreakdownDimension, - run_id: Optional[str] = None, - filters: Optional[dict[str, Union[str, list[str]]]] = None, - ) -> BenchmarkStandingsBreakdown: - """ - Returns the benchmark standings broken down by a demographic dimension. - - Alongside the global standings, this returns one segment per bucket of - ``dimension`` (including an ``"unknown"`` bucket), each with that - segment's raw vote count and its own standings. This shows how different - voter groups rank the models relative to each other. - - Segments are **raw vote counts**. Note that the ``AgeBucket``, ``Gender`` - and ``Occupation`` dimensions are **estimated** (inferred), not - self-declared. - - Args: - benchmark_id: The id of the benchmark. - dimension: The demographic dimension to break the standings down by. One of "AgeBucket", "Gender", "Occupation", "Country", "Language". - run_id: Restrict the breakdown to a single evaluation run. If None, all runs are considered. - filters: Additional filters on the underlying votes, keyed by field name (same fields as :py:meth:`get_demographics`). Each value is a string or list of strings matched with ``in`` semantics. - - Returns: - A BenchmarkStandingsBreakdown with the global standings and per-segment standings. - """ - with tracer.start_as_current_span( - "RapidataBenchmarkManager.get_standings_breakdown" - ): - if dimension not in BreakdownDimension.__args__: - raise ValueError( - "Dimension must be one of: " - + ", ".join(BreakdownDimension.__args__) - ) - - return self.__openapi_service.leaderboard.benchmark_demographics_api.get_standings_breakdown( - benchmark_id=benchmark_id, - dimension=dimension, - filters=self.__build_filters(filters, run_id), - ) - - @staticmethod - def __build_filters( - filters: Optional[dict[str, Union[str, list[str]]]], - run_id: Optional[str], - ) -> dict[str, AudienceAudienceIdJobsGetJobIdParameter]: - raw: dict[str, Union[str, list[str]]] = dict(filters or {}) - if run_id is not None: - raw["run_id"] = run_id - - built: dict[str, AudienceAudienceIdJobsGetJobIdParameter] = {} - for field, value in raw.items(): - if field not in BENCHMARK_FILTER_FIELDS: - raise ValueError( - f"Unknown filter field '{field}'. Allowed fields: " - + ", ".join(BENCHMARK_FILTER_FIELDS) - ) - values = [value] if isinstance(value, str) else list(value) - param = AudienceAudienceIdJobsGetJobIdParameter() - param.var_in = values - built[field] = param - return built From a05547e2dbc2c02fd58b9d460fbcbe2a1cd0efd4 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Thu, 23 Jul 2026 14:37:59 +0000 Subject: [PATCH 5/7] refactor(benchmark): type the breakdown dimension with the generated enum Address review (@LinoGiger): use the BenchmarkDemographicDimension enum the generated client already provides instead of a bare string literal. - get_standings_breakdown now takes BenchmarkDemographicDimension; a raw string is still coerced for convenience and an invalid value raises a clear ValueError. - Re-export BenchmarkDemographicDimension from the top-level package. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: luca <25279790+LucStr@users.noreply.github.com> --- docs/mri_advanced.md | 9 +++++--- src/rapidata/__init__.py | 1 + src/rapidata/rapidata_client/__init__.py | 3 +++ .../benchmark/rapidata_benchmark.py | 22 +++++-------------- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index eb5a93cb0..e7512397f 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -275,11 +275,14 @@ print(countries) `get_standings_breakdown` returns one row per `(segment, model)` — how each demographic segment of voters ranks the models, with that segment's vote count. Useful for spotting where a model is ranked differently by different groups. For -the overall standings across all voters, use `get_overall_standings`. +the overall standings across all voters, use `get_overall_standings`. The +dimension is a `BenchmarkDemographicDimension`. ```python +from rapidata import BenchmarkDemographicDimension + breakdown = benchmark.get_standings_breakdown( - dimension="AgeBucket", # AgeBucket | Gender | Occupation | Country | Language + dimension=BenchmarkDemographicDimension.AGEBUCKET, ) # columns: segment, segment_votes, name, wins, total_matches, score ``` @@ -289,7 +292,7 @@ and `run_id` to narrow the votes considered: ```python breakdown = benchmark.get_standings_breakdown( - dimension="Country", + dimension=BenchmarkDemographicDimension.COUNTRY, tags=["landscape"], ) ``` diff --git a/src/rapidata/__init__.py b/src/rapidata/__init__.py index 0e3909325..34f7074d3 100644 --- a/src/rapidata/__init__.py +++ b/src/rapidata/__init__.py @@ -20,6 +20,7 @@ RapidataValidationSet, Box, RapidataResults, + BenchmarkDemographicDimension, DemographicSelection, LabelingSelection, EffortSelection, diff --git a/src/rapidata/rapidata_client/__init__.py b/src/rapidata/rapidata_client/__init__.py index 0ebd59f92..9aa3d2723 100644 --- a/src/rapidata/rapidata_client/__init__.py +++ b/src/rapidata/rapidata_client/__init__.py @@ -17,6 +17,9 @@ from .signal import RapidataSignal, RapidataSignalManager from .validation import ValidationSetManager, RapidataValidationSet, Box from .results import RapidataResults +from rapidata.api_client.models.benchmark_demographic_dimension import ( + BenchmarkDemographicDimension, +) from .selection import ( DemographicSelection, LabelingSelection, diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 321d068c2..f91ff0ba2 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -14,6 +14,9 @@ from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( AudienceAudienceIdJobsGetJobIdParameter, ) +from rapidata.api_client.models.benchmark_demographic_dimension import ( + BenchmarkDemographicDimension, +) if TYPE_CHECKING: import pandas as pd @@ -27,11 +30,6 @@ from rapidata.rapidata_client.settings import RapidataSetting from rapidata.service.openapi_service import OpenAPIService -# The demographic dimension to split standings by, as accepted by the -# breakdown endpoint. Age, gender and occupation are estimated (inferred); -# country and language are observed. -BreakdownDimension = Literal["AgeBucket", "Gender", "Occupation", "Country", "Language"] - class RapidataBenchmark: """ @@ -881,7 +879,7 @@ def get_demographics( def get_standings_breakdown( self, - dimension: BreakdownDimension, + dimension: BenchmarkDemographicDimension, tags: Optional[list[str]] = None, leaderboard_ids: Optional[list[str]] = None, run_id: Optional[str] = None, @@ -898,7 +896,7 @@ def get_standings_breakdown( self-declared; ``Country`` and ``Language`` are observed. Args: - dimension: The demographic dimension to split by. One of "AgeBucket", "Gender", "Occupation", "Country", "Language". + dimension: The :class:`BenchmarkDemographicDimension` to split by (``AgeBucket``, ``Gender``, ``Occupation``, ``Country`` or ``Language``). tags: Only count votes on matchups with these tags. If None, all matchups are considered. leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered. run_id: Only count votes from this evaluation run. If None, all runs are considered. @@ -906,19 +904,9 @@ def get_standings_breakdown( Returns: A pandas DataFrame with columns ``segment``, ``segment_votes``, ``name``, ``wins``, ``total_matches``, ``score``. """ - from rapidata.api_client.models.benchmark_demographic_dimension import ( - BenchmarkDemographicDimension, - ) - import pandas as pd with tracer.start_as_current_span("RapidataBenchmark.get_standings_breakdown"): - if dimension not in BreakdownDimension.__args__: - raise ValueError( - "Dimension must be one of: " - + ", ".join(BreakdownDimension.__args__) - ) - tags_filter, leaderboard_filter, run_filter = self.__demographic_filters( tags, leaderboard_ids, run_id ) From 7f1668bef92d0c03ca781b96badd8c6b75c9ed39 Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Thu, 23 Jul 2026 14:40:38 +0000 Subject: [PATCH 6/7] refactor(benchmark): report demographics dimension as the typed enum value Make get_demographics speak the same BenchmarkDemographicDimension vocabulary as get_standings_breakdown: the DataFrame 'dimension' column now holds the enum's values (AgeBucket, Gender, ...) instead of raw camelCase strings. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: luca <25279790+LucStr@users.noreply.github.com> --- docs/mri_advanced.md | 10 ++++++++-- .../benchmark/rapidata_benchmark.py | 14 +++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index e7512397f..ecabb13a2 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -260,13 +260,19 @@ the benchmark and return a pandas DataFrame, like `get_overall_standings`. ### Voter composition `get_demographics` returns one row per `(dimension, value)` with the raw vote -count and its share of the dimension (shares within a dimension sum to 1): +count and its share of the dimension (shares within a dimension sum to 1). The +`dimension` column holds `BenchmarkDemographicDimension` values (`AgeBucket`, +`Gender`, `Occupation`, `Country`, `Language`): ```python +from rapidata import BenchmarkDemographicDimension + demographics = benchmark.get_demographics() # columns: dimension, value, votes, share -countries = demographics[demographics["dimension"] == "country"] +countries = demographics[ + demographics["dimension"] == BenchmarkDemographicDimension.COUNTRY +] print(countries) ``` diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index f91ff0ba2..1c3a25943 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -856,20 +856,20 @@ def get_demographics( dimensions = result.dimensions dimension_buckets = { - "ageBucket": dimensions.age_bucket, - "gender": dimensions.gender, - "occupation": dimensions.occupation, - "country": dimensions.country, - "language": dimensions.language, + BenchmarkDemographicDimension.AGEBUCKET: dimensions.age_bucket, + BenchmarkDemographicDimension.GENDER: dimensions.gender, + BenchmarkDemographicDimension.OCCUPATION: dimensions.occupation, + BenchmarkDemographicDimension.COUNTRY: dimensions.country, + BenchmarkDemographicDimension.LANGUAGE: dimensions.language, } rows = [ { - "dimension": name, + "dimension": dimension.value, "value": bucket.value, "votes": bucket.votes, "share": bucket.share, } - for name, buckets in dimension_buckets.items() + for dimension, buckets in dimension_buckets.items() for bucket in buckets ] From 1fa4573b6b005429c968ccd24331f9b23353ea9e Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Thu, 23 Jul 2026 15:12:29 +0000 Subject: [PATCH 7/7] feat(benchmark): filter standings and matrices by voter demographics Address review (@LinoGiger): the standings and matrix reads were missing the demographic filters. Add country / language / gender / age_bucket / occupation / run_id to every benchmark and leaderboard read that hits the vote-query endpoints: - RapidataBenchmark.get_overall_standings, get_win_loss_matrix, get_demographics, get_standings_breakdown - RapidataLeaderboard.get_standings, get_win_loss_matrix The filters compute the result from only the votes cast by matching voters (e.g. standings among young women). gender / age_bucket take the SDK's typed Gender / AgeGroup enums; the open-ended dimensions take plain values. A shared _vote_filters helper converts them to the backend filter shape. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: luca <25279790+LucStr@users.noreply.github.com> --- docs/mri_advanced.md | 25 ++- .../benchmark/_vote_filters.py | 64 +++++++ .../leaderboard/rapidata_leaderboard.py | 72 +++++++- .../benchmark/rapidata_benchmark.py | 163 ++++++++++++------ 4 files changed, 260 insertions(+), 64 deletions(-) create mode 100644 src/rapidata/rapidata_client/benchmark/_vote_filters.py diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index ecabb13a2..4278b4aff 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -293,16 +293,37 @@ breakdown = benchmark.get_standings_breakdown( # columns: segment, segment_votes, name, wins, total_matches, score ``` -Segments are **raw vote counts**. Both methods accept `tags`, `leaderboard_ids` -and `run_id` to narrow the votes considered: +Segments are **raw vote counts**. + +### Filtering standings and matrices by demographics + +The demographic dimensions are also filters. `get_overall_standings`, +`get_win_loss_matrix`, `get_demographics` and `get_standings_breakdown` on the +benchmark — and `get_standings` / `get_win_loss_matrix` on a leaderboard — all +accept `country`, `language`, `gender`, `age_bucket`, `occupation` and `run_id` +(on top of `tags` / `leaderboard_ids`). This computes the result from only the +votes cast by matching voters, e.g. the standings among young women: ```python +from rapidata import BenchmarkDemographicDimension, Gender, AgeGroup + +standings = benchmark.get_overall_standings( + gender=[Gender.FEMALE], + age_bucket=[AgeGroup.BETWEEN_18_29], + country=["US", "GB"], +) + +# and any of the other read methods, e.g. breakdown = benchmark.get_standings_breakdown( dimension=BenchmarkDemographicDimension.COUNTRY, tags=["landscape"], + gender=[Gender.FEMALE], ) ``` +`gender` and `age_bucket` take the SDK's `Gender` / `AgeGroup` enums; `country`, +`language` and `occupation` take plain values. + ## Win/Loss Matrix `get_standings` collapses every matchup into one Elo score per model. When you want diff --git a/src/rapidata/rapidata_client/benchmark/_vote_filters.py b/src/rapidata/rapidata_client/benchmark/_vote_filters.py new file mode 100644 index 000000000..a9daf67c1 --- /dev/null +++ b/src/rapidata/rapidata_client/benchmark/_vote_filters.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import NamedTuple, Optional, TYPE_CHECKING + +from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( + AudienceAudienceIdJobsGetJobIdParameter, +) + +if TYPE_CHECKING: + from rapidata.rapidata_client.filter.models.gender import Gender + from rapidata.rapidata_client.filter.models.age_group import AgeGroup + + +def in_filter( + values: Optional[list[str]], +) -> AudienceAudienceIdJobsGetJobIdParameter: + """Wrap a list of accepted values as an ``in`` filter (a no-op when None).""" + param = AudienceAudienceIdJobsGetJobIdParameter() + param.var_in = values + return param + + +class DemographicFilters(NamedTuple): + """The per-vote demographic filters shared by the benchmark and leaderboard + standings, matrix, demographics and standings-breakdown endpoints. + + Each field is keyed to match the corresponding query-endpoint parameter, so a + caller passes them straight through (``country=filters.country`` ...). + """ + + country: AudienceAudienceIdJobsGetJobIdParameter + language: AudienceAudienceIdJobsGetJobIdParameter + gender: AudienceAudienceIdJobsGetJobIdParameter + age_bucket: AudienceAudienceIdJobsGetJobIdParameter + occupation: AudienceAudienceIdJobsGetJobIdParameter + run_id: AudienceAudienceIdJobsGetJobIdParameter + + +def demographic_filters( + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, +) -> DemographicFilters: + """Build the demographic filters for a standings/matrix/demographics call. + + ``gender`` and ``age_bucket`` take the SDK's :class:`Gender` and :class:`AgeGroup` + enums and are converted to their backend values; the open-ended dimensions are + plain values. Unset filters become no-ops. + """ + return DemographicFilters( + country=in_filter(country), + language=in_filter(language), + gender=in_filter( + [g._to_backend_model().value for g in gender] if gender else None + ), + age_bucket=in_filter( + [a._to_backend_model().value for a in age_bucket] if age_bucket else None + ), + occupation=in_filter(occupation), + run_id=in_filter([run_id] if run_id is not None else None), + ) diff --git a/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py b/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py index ec3c5f8be..8ac164162 100644 --- a/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py +++ b/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py @@ -10,8 +10,9 @@ LevelOfDetail, ResolvedLevelOfDetail, ) -from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( - AudienceAudienceIdJobsGetJobIdParameter, +from rapidata.rapidata_client.benchmark._vote_filters import ( + demographic_filters, + in_filter, ) from rapidata.service.openapi_service import OpenAPIService from rapidata.api_client.models.update_leaderboard_endpoint_input import ( @@ -21,6 +22,8 @@ if TYPE_CHECKING: import pandas as pd from rapidata.rapidata_client.job.rapidata_job import RapidataJob + from rapidata.rapidata_client.filter.models.gender import Gender + from rapidata.rapidata_client.filter.models.age_group import AgeGroup class RapidataLeaderboard: @@ -224,23 +227,51 @@ def jobs(self) -> list[RapidataJob]: return [job_manager.get_job_by_id(job_id) for job_id in job_ids] - def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame": + def get_standings( + self, + tags: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, + ) -> "pd.DataFrame": """ Returns the standings of the leaderboard. + The demographic filters compute the standings from only the votes cast by + matching voters — e.g. the standings among women, or among US voters. + ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` and + ``language`` are observed. + Args: tags: The matchups with these tags should be used to create the standings. If tags are None, all matchups will be considered. If tags are empty, no matchups will be considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. Returns: A pandas DataFrame containing the standings of the leaderboard. """ with tracer.start_as_current_span("RapidataLeaderboard.get_standings"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) participants = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_standings_query_get( - leaderboard_id=self.id, tags=tags_filter + leaderboard_id=self.id, + tags=in_filter(tags), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) import pandas as pd @@ -266,6 +297,12 @@ def get_win_loss_matrix( self, tags: Optional[list[str]] = None, use_weighted_scoring: Optional[bool] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, ) -> pd.DataFrame: """ Returns the pairwise win/loss matrix for the participants in this leaderboard. @@ -277,6 +314,10 @@ def get_win_loss_matrix( always 0. This is the head-to-head breakdown behind :meth:`get_standings`, which collapses the same matchups into a single Elo score per model. + The demographic filters restrict the matrix to matchups decided by matching + voters. ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` + and ``language`` are observed. + Args: tags: Only count matchups carrying one of these prompt tags. If None, every matchup on the leaderboard is included; if an empty list, none are. @@ -285,6 +326,12 @@ def get_win_loss_matrix( plain win, so cells hold weighted sums (floats) rather than raw counts. If False, cells are raw win counts. When None (default), the server applies the leaderboard's configured default. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. Returns: A pandas DataFrame indexed by participant name on both axes, where cell @@ -292,12 +339,19 @@ def get_win_loss_matrix( participant over the column participant. """ with tracer.start_as_current_span("RapidataLeaderboard.get_win_loss_matrix"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) result = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_matrix_query_get( leaderboard_id=self.id, - tags=tags_filter, + tags=in_filter(tags), use_weighted_scoring=use_weighted_scoring, + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) import pandas as pd diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 1c3a25943..2aff07082 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -11,12 +11,13 @@ BenchmarkPrompt, BenchmarkPromptUploader, ) -from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( - AudienceAudienceIdJobsGetJobIdParameter, -) from rapidata.api_client.models.benchmark_demographic_dimension import ( BenchmarkDemographicDimension, ) +from rapidata.rapidata_client.benchmark._vote_filters import ( + demographic_filters, + in_filter, +) if TYPE_CHECKING: import pandas as pd @@ -27,6 +28,8 @@ from rapidata.rapidata_client.benchmark.participant.participant import ( BenchmarkParticipant, ) + from rapidata.rapidata_client.filter.models.gender import Gender + from rapidata.rapidata_client.filter.models.age_group import AgeGroup from rapidata.rapidata_client.settings import RapidataSetting from rapidata.service.openapi_service import OpenAPIService @@ -716,26 +719,47 @@ def get_overall_standings( self, tags: Optional[list[str]] = None, leaderboard_ids: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, ) -> pd.DataFrame: """ Returns an aggregated elo table of all leaderboards in the benchmark. + The demographic filters compute the standings from only the votes cast by + matching voters — e.g. the standings among women, or among US voters. + ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` and + ``language`` are observed. + Args: tags: Filter standings by these tags. If None, all tags are considered. leaderboard_ids: Filter to only include matchups from these leaderboards. If None, all leaderboards are considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. """ import pandas as pd with tracer.start_as_current_span("get_overall_standings"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags - leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter() - leaderboard_filter.var_in = leaderboard_ids - + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) participants = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_standings_query_get( benchmark_id=self.id, - tags=tags_filter, - leaderboard_id=leaderboard_filter, + tags=in_filter(tags), + leaderboard_id=in_filter(leaderboard_ids), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) standings = [] @@ -761,6 +785,12 @@ def get_win_loss_matrix( participant_ids: Optional[list[str]] = None, leaderboard_ids: Optional[list[str]] = None, use_weighted_scoring: Optional[bool] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, ) -> pd.DataFrame: """ Returns the pairwise win/loss matrix aggregated across the benchmark's leaderboards. @@ -773,6 +803,10 @@ def get_win_loss_matrix( head-to-head breakdown behind :meth:`get_overall_standings`, which collapses the same matchups into a single Elo score per model. + The demographic filters restrict the matrix to matchups decided by matching + voters. ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` + and ``language`` are observed. + Args: tags: Only count matchups carrying one of these prompt tags. If None, every matchup is included; if an empty list, none are. @@ -785,6 +819,12 @@ def get_win_loss_matrix( plain win, so cells hold weighted sums (floats) rather than raw counts. If False, cells are raw win counts. When None (default), the server applies its configured default. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. Returns: A pandas DataFrame indexed by participant name on both axes, where cell @@ -794,19 +834,21 @@ def get_win_loss_matrix( import pandas as pd with tracer.start_as_current_span("get_win_loss_matrix"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags - participant_filter = AudienceAudienceIdJobsGetJobIdParameter() - participant_filter.var_in = participant_ids - leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter() - leaderboard_filter.var_in = leaderboard_ids - + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_matrix_query_get( benchmark_id=self.id, - tags=tags_filter, - participant_id=participant_filter, - leaderboard_id=leaderboard_filter, + tags=in_filter(tags), + participant_id=in_filter(participant_ids), + leaderboard_id=in_filter(leaderboard_ids), use_weighted_scoring=use_weighted_scoring, + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) return pd.DataFrame( @@ -819,6 +861,11 @@ def get_demographics( self, tags: Optional[list[str]] = None, leaderboard_ids: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, run_id: Optional[str] = None, ) -> pd.DataFrame: """ @@ -826,16 +873,22 @@ def get_demographics( One row per (dimension, bucket): the ``votes`` cast by that bucket and its ``share`` of the dimension's votes (a dimension's shares sum to 1). - Every dimension (``ageBucket``, ``gender``, ``occupation``, ``country``, - ``language``) includes an ``"unknown"`` bucket for votes whose attribute + Every dimension (``AgeBucket``, ``Gender``, ``Occupation``, ``Country``, + ``Language``) includes an ``"unknown"`` bucket for votes whose attribute could not be determined. - ``ageBucket``, ``gender`` and ``occupation`` are estimated (inferred from - behaviour), not self-declared; ``country`` and ``language`` are observed. + ``AgeBucket``, ``Gender`` and ``Occupation`` are estimated (inferred from + behaviour), not self-declared; ``Country`` and ``Language`` are observed. + The demographic filters restrict the composition to matching voters. Args: tags: Only count votes on matchups with these tags. If None, all matchups are considered. leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. run_id: Only count votes from this evaluation run. If None, all runs are considered. Returns: @@ -844,14 +897,19 @@ def get_demographics( import pandas as pd with tracer.start_as_current_span("RapidataBenchmark.get_demographics"): - tags_filter, leaderboard_filter, run_filter = self.__demographic_filters( - tags, leaderboard_ids, run_id + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id ) result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_demographics_get( benchmark_id=self.id, - tags=tags_filter, - leaderboard_id=leaderboard_filter, - run_id=run_filter, + tags=in_filter(tags), + leaderboard_id=in_filter(leaderboard_ids), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) dimensions = result.dimensions @@ -882,6 +940,11 @@ def get_standings_breakdown( dimension: BenchmarkDemographicDimension, tags: Optional[list[str]] = None, leaderboard_ids: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, run_id: Optional[str] = None, ) -> pd.DataFrame: """ @@ -893,12 +956,19 @@ def get_standings_breakdown( overall standings across all voters, use :py:meth:`get_overall_standings`. ``AgeBucket``, ``Gender`` and ``Occupation`` are estimated (inferred), not - self-declared; ``Country`` and ``Language`` are observed. + self-declared; ``Country`` and ``Language`` are observed. The demographic + filters narrow the voters before the split (e.g. break down by age within + US voters only). Args: dimension: The :class:`BenchmarkDemographicDimension` to split by (``AgeBucket``, ``Gender``, ``Occupation``, ``Country`` or ``Language``). tags: Only count votes on matchups with these tags. If None, all matchups are considered. leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. run_id: Only count votes from this evaluation run. If None, all runs are considered. Returns: @@ -907,15 +977,20 @@ def get_standings_breakdown( import pandas as pd with tracer.start_as_current_span("RapidataBenchmark.get_standings_breakdown"): - tags_filter, leaderboard_filter, run_filter = self.__demographic_filters( - tags, leaderboard_ids, run_id + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id ) result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_standings_breakdown_get( benchmark_id=self.id, dimension=BenchmarkDemographicDimension(dimension), - tags=tags_filter, - leaderboard_id=leaderboard_filter, - run_id=run_filter, + tags=in_filter(tags), + leaderboard_id=in_filter(leaderboard_ids), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) rows = [] @@ -948,24 +1023,6 @@ def get_standings_breakdown( ), ) - def __demographic_filters( - self, - tags: Optional[list[str]], - leaderboard_ids: Optional[list[str]], - run_id: Optional[str], - ) -> tuple[ - AudienceAudienceIdJobsGetJobIdParameter, - AudienceAudienceIdJobsGetJobIdParameter, - AudienceAudienceIdJobsGetJobIdParameter, - ]: - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags - leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter() - leaderboard_filter.var_in = leaderboard_ids - run_filter = AudienceAudienceIdJobsGetJobIdParameter() - run_filter.var_in = [run_id] if run_id is not None else None - return tags_filter, leaderboard_filter, run_filter - def __str__(self) -> str: return f"RapidataBenchmark(name={self.name}, id={self.id})"