feat(benchmark): expose demographics and standings breakdown in SDK

docs/mri_advanced.md

@@ -245,6 +245,55 @@ participant = benchmark.participants[0]
 participant.delete()
 ```
 
+## Voter Demographics
+
+Every vote on a benchmark carries the voter's demographics, so you can see *who*
+evaluated your models and how different groups ranked them. Both methods live on
+the benchmark and return a pandas DataFrame, like `get_overall_standings`.
+
+!!! note
+    `ageBucket`, `gender` and `occupation` are **estimated** (inferred from
+    behaviour), not self-declared. `country` and `language` are observed. Each
+    dimension includes an `"unknown"` bucket for votes whose attribute could not
+    be determined.
+
+### Voter composition
+
+`get_demographics` returns one row per `(dimension, value)` with the raw vote
+count and its share of the dimension (shares within a dimension sum to 1):
+
+```python
+demographics = benchmark.get_demographics()
+# columns: dimension, value, votes, share
+
+countries = demographics[demographics["dimension"] == "country"]
+print(countries)
+```
+
+### Standings per demographic segment
+
+`get_standings_breakdown` returns one row per `(segment, model)` — how each
+demographic segment of voters ranks the models, with that segment's vote count.
+Useful for spotting where a model is ranked differently by different groups. For
+the overall standings across all voters, use `get_overall_standings`.
+
+```python
+breakdown = benchmark.get_standings_breakdown(
+    dimension="AgeBucket",  # AgeBucket | Gender | Occupation | Country | Language
+)
+# columns: segment, segment_votes, name, wins, total_matches, score
+```
+
+Segments are **raw vote counts**. Both methods accept `tags`, `leaderboard_ids`
+and `run_id` to narrow the votes considered:
+
+```python
+breakdown = benchmark.get_standings_breakdown(
+    dimension="Country",
+    tags=["landscape"],
+)
+```
+
 ## Win/Loss Matrix
 
 `get_standings` collapses every matchup into one Elo score per model. When you want

src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py

@@ -27,6 +27,11 @@
     from rapidata.rapidata_client.settings import RapidataSetting
     from rapidata.service.openapi_service import OpenAPIService
 
+# The demographic dimension to split standings by, as accepted by the
+# breakdown endpoint. Age, gender and occupation are estimated (inferred);
+# country and language are observed.
+BreakdownDimension = Literal["AgeBucket", "Gender", "Occupation", "Country", "Language"]
+
 
 class RapidataBenchmark:
     """
@@ -812,6 +817,167 @@ def get_win_loss_matrix(
                 columns=pd.Index(result.columns),
             )
 
+    def get_demographics(
+        self,
+        tags: Optional[list[str]] = None,
+        leaderboard_ids: Optional[list[str]] = None,
+        run_id: Optional[str] = None,
+    ) -> pd.DataFrame:
+        """
+        Returns the demographic composition of the voters for this benchmark.
+
+        One row per (dimension, bucket): the ``votes`` cast by that bucket and
+        its ``share`` of the dimension's votes (a dimension's shares sum to 1).
+        Every dimension (``ageBucket``, ``gender``, ``occupation``, ``country``,
+        ``language``) includes an ``"unknown"`` bucket for votes whose attribute
+        could not be determined.
+
+        ``ageBucket``, ``gender`` and ``occupation`` are estimated (inferred from
+        behaviour), not self-declared; ``country`` and ``language`` are observed.
+
+        Args:
+            tags: Only count votes on matchups with these tags. If None, all matchups are considered.
+            leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered.
+            run_id: Only count votes from this evaluation run. If None, all runs are considered.
+
+        Returns:
+            A pandas DataFrame with columns ``dimension``, ``value``, ``votes``, ``share``.
+        """
+        import pandas as pd
+
+        with tracer.start_as_current_span("RapidataBenchmark.get_demographics"):
+            tags_filter, leaderboard_filter, run_filter = self.__demographic_filters(
+                tags, leaderboard_ids, run_id
+            )
+            result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_demographics_get(
+                benchmark_id=self.id,
+                tags=tags_filter,
+                leaderboard_id=leaderboard_filter,
+                run_id=run_filter,
+            )
+
+            dimensions = result.dimensions
+            dimension_buckets = {
+                "ageBucket": dimensions.age_bucket,
+                "gender": dimensions.gender,
+                "occupation": dimensions.occupation,
+                "country": dimensions.country,
+                "language": dimensions.language,
+            }
+            rows = [
+                {
+                    "dimension": name,
+                    "value": bucket.value,
+                    "votes": bucket.votes,
+                    "share": bucket.share,
+                }
+                for name, buckets in dimension_buckets.items()
+                for bucket in buckets
+            ]
+
+            return pd.DataFrame(
+                rows, columns=pd.Index(["dimension", "value", "votes", "share"])
+            )
+
+    def get_standings_breakdown(
+        self,
+        dimension: BreakdownDimension,
+        tags: Optional[list[str]] = None,
+        leaderboard_ids: Optional[list[str]] = None,
+        run_id: Optional[str] = None,
+    ) -> pd.DataFrame:
+        """
+        Returns the standings split by a demographic dimension of the voters.
+
+        One row per (segment, model): how each demographic segment of voters
+        ranks the models, alongside that segment's vote count. The segments
+        include an ``"unknown"`` bucket. Scores are raw vote counts. For the
+        overall standings across all voters, use :py:meth:`get_overall_standings`.
+
+        ``AgeBucket``, ``Gender`` and ``Occupation`` are estimated (inferred), not
+        self-declared; ``Country`` and ``Language`` are observed.
+
+        Args:
+            dimension: The demographic dimension to split by. One of "AgeBucket", "Gender", "Occupation", "Country", "Language".
+            tags: Only count votes on matchups with these tags. If None, all matchups are considered.
+            leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered.
+            run_id: Only count votes from this evaluation run. If None, all runs are considered.
+
+        Returns:
+            A pandas DataFrame with columns ``segment``, ``segment_votes``, ``name``, ``wins``, ``total_matches``, ``score``.
+        """
+        from rapidata.api_client.models.benchmark_demographic_dimension import (
+            BenchmarkDemographicDimension,
+        )
+
+        import pandas as pd
+
+        with tracer.start_as_current_span("RapidataBenchmark.get_standings_breakdown"):
+            if dimension not in BreakdownDimension.__args__:
+                raise ValueError(
+                    "Dimension must be one of: "
+                    + ", ".join(BreakdownDimension.__args__)
+                )
+
+            tags_filter, leaderboard_filter, run_filter = self.__demographic_filters(
+                tags, leaderboard_ids, run_id
+            )
+            result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_standings_breakdown_get(
+                benchmark_id=self.id,
+                dimension=BenchmarkDemographicDimension(dimension),
+                tags=tags_filter,
+                leaderboard_id=leaderboard_filter,
+                run_id=run_filter,
+            )
+
+            rows = []
+            for segment in result.segments:
+                for item in segment.items:
+                    rows.append(
+                        {
+                            "segment": segment.value,
+                            "segment_votes": segment.votes,
+                            "name": item.name,
+                            "wins": item.wins,
+                            "total_matches": item.total_matches,
+                            "score": (
+                                round(item.score, 2) if item.score is not None else None
+                            ),
+                        }
+                    )
+
+            return pd.DataFrame(
+                rows,
+                columns=pd.Index(
+                    [
+                        "segment",
+                        "segment_votes",
+                        "name",
+                        "wins",
+                        "total_matches",
+                        "score",
+                    ]
+                ),
+            )
+
+    def __demographic_filters(
+        self,
+        tags: Optional[list[str]],
+        leaderboard_ids: Optional[list[str]],
+        run_id: Optional[str],
+    ) -> tuple[
+        AudienceAudienceIdJobsGetJobIdParameter,
+        AudienceAudienceIdJobsGetJobIdParameter,
+        AudienceAudienceIdJobsGetJobIdParameter,
+    ]:
+        tags_filter = AudienceAudienceIdJobsGetJobIdParameter()
+        tags_filter.var_in = tags
+        leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter()
+        leaderboard_filter.var_in = leaderboard_ids
+        run_filter = AudienceAudienceIdJobsGetJobIdParameter()
+        run_filter.var_in = [run_id] if run_id is not None else None
+        return tags_filter, leaderboard_filter, run_filter
+
     def __str__(self) -> str:
         return f"RapidataBenchmark(name={self.name}, id={self.id})"