diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index d52483f4c..4278b4aff 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -245,6 +245,85 @@ participant = benchmark.participants[0] participant.delete() ``` +## Voter Demographics + +Every vote on a benchmark carries the voter's demographics, so you can see *who* +evaluated your models and how different groups ranked them. Both methods live on +the benchmark and return a pandas DataFrame, like `get_overall_standings`. + +!!! note + `ageBucket`, `gender` and `occupation` are **estimated** (inferred from + behaviour), not self-declared. `country` and `language` are observed. Each + dimension includes an `"unknown"` bucket for votes whose attribute could not + be determined. + +### Voter composition + +`get_demographics` returns one row per `(dimension, value)` with the raw vote +count and its share of the dimension (shares within a dimension sum to 1). The +`dimension` column holds `BenchmarkDemographicDimension` values (`AgeBucket`, +`Gender`, `Occupation`, `Country`, `Language`): + +```python +from rapidata import BenchmarkDemographicDimension + +demographics = benchmark.get_demographics() +# columns: dimension, value, votes, share + +countries = demographics[ + demographics["dimension"] == BenchmarkDemographicDimension.COUNTRY +] +print(countries) +``` + +### Standings per demographic segment + +`get_standings_breakdown` returns one row per `(segment, model)` — how each +demographic segment of voters ranks the models, with that segment's vote count. +Useful for spotting where a model is ranked differently by different groups. For +the overall standings across all voters, use `get_overall_standings`. The +dimension is a `BenchmarkDemographicDimension`. + +```python +from rapidata import BenchmarkDemographicDimension + +breakdown = benchmark.get_standings_breakdown( + dimension=BenchmarkDemographicDimension.AGEBUCKET, +) +# columns: segment, segment_votes, name, wins, total_matches, score +``` + +Segments are **raw vote counts**. + +### Filtering standings and matrices by demographics + +The demographic dimensions are also filters. `get_overall_standings`, +`get_win_loss_matrix`, `get_demographics` and `get_standings_breakdown` on the +benchmark — and `get_standings` / `get_win_loss_matrix` on a leaderboard — all +accept `country`, `language`, `gender`, `age_bucket`, `occupation` and `run_id` +(on top of `tags` / `leaderboard_ids`). This computes the result from only the +votes cast by matching voters, e.g. the standings among young women: + +```python +from rapidata import BenchmarkDemographicDimension, Gender, AgeGroup + +standings = benchmark.get_overall_standings( + gender=[Gender.FEMALE], + age_bucket=[AgeGroup.BETWEEN_18_29], + country=["US", "GB"], +) + +# and any of the other read methods, e.g. +breakdown = benchmark.get_standings_breakdown( + dimension=BenchmarkDemographicDimension.COUNTRY, + tags=["landscape"], + gender=[Gender.FEMALE], +) +``` + +`gender` and `age_bucket` take the SDK's `Gender` / `AgeGroup` enums; `country`, +`language` and `occupation` take plain values. + ## Win/Loss Matrix `get_standings` collapses every matchup into one Elo score per model. When you want diff --git a/src/rapidata/__init__.py b/src/rapidata/__init__.py index 0e3909325..34f7074d3 100644 --- a/src/rapidata/__init__.py +++ b/src/rapidata/__init__.py @@ -20,6 +20,7 @@ RapidataValidationSet, Box, RapidataResults, + BenchmarkDemographicDimension, DemographicSelection, LabelingSelection, EffortSelection, diff --git a/src/rapidata/rapidata_client/__init__.py b/src/rapidata/rapidata_client/__init__.py index 0ebd59f92..9aa3d2723 100644 --- a/src/rapidata/rapidata_client/__init__.py +++ b/src/rapidata/rapidata_client/__init__.py @@ -17,6 +17,9 @@ from .signal import RapidataSignal, RapidataSignalManager from .validation import ValidationSetManager, RapidataValidationSet, Box from .results import RapidataResults +from rapidata.api_client.models.benchmark_demographic_dimension import ( + BenchmarkDemographicDimension, +) from .selection import ( DemographicSelection, LabelingSelection, diff --git a/src/rapidata/rapidata_client/benchmark/_vote_filters.py b/src/rapidata/rapidata_client/benchmark/_vote_filters.py new file mode 100644 index 000000000..a9daf67c1 --- /dev/null +++ b/src/rapidata/rapidata_client/benchmark/_vote_filters.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import NamedTuple, Optional, TYPE_CHECKING + +from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( + AudienceAudienceIdJobsGetJobIdParameter, +) + +if TYPE_CHECKING: + from rapidata.rapidata_client.filter.models.gender import Gender + from rapidata.rapidata_client.filter.models.age_group import AgeGroup + + +def in_filter( + values: Optional[list[str]], +) -> AudienceAudienceIdJobsGetJobIdParameter: + """Wrap a list of accepted values as an ``in`` filter (a no-op when None).""" + param = AudienceAudienceIdJobsGetJobIdParameter() + param.var_in = values + return param + + +class DemographicFilters(NamedTuple): + """The per-vote demographic filters shared by the benchmark and leaderboard + standings, matrix, demographics and standings-breakdown endpoints. + + Each field is keyed to match the corresponding query-endpoint parameter, so a + caller passes them straight through (``country=filters.country`` ...). + """ + + country: AudienceAudienceIdJobsGetJobIdParameter + language: AudienceAudienceIdJobsGetJobIdParameter + gender: AudienceAudienceIdJobsGetJobIdParameter + age_bucket: AudienceAudienceIdJobsGetJobIdParameter + occupation: AudienceAudienceIdJobsGetJobIdParameter + run_id: AudienceAudienceIdJobsGetJobIdParameter + + +def demographic_filters( + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, +) -> DemographicFilters: + """Build the demographic filters for a standings/matrix/demographics call. + + ``gender`` and ``age_bucket`` take the SDK's :class:`Gender` and :class:`AgeGroup` + enums and are converted to their backend values; the open-ended dimensions are + plain values. Unset filters become no-ops. + """ + return DemographicFilters( + country=in_filter(country), + language=in_filter(language), + gender=in_filter( + [g._to_backend_model().value for g in gender] if gender else None + ), + age_bucket=in_filter( + [a._to_backend_model().value for a in age_bucket] if age_bucket else None + ), + occupation=in_filter(occupation), + run_id=in_filter([run_id] if run_id is not None else None), + ) diff --git a/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py b/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py index ec3c5f8be..8ac164162 100644 --- a/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py +++ b/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py @@ -10,8 +10,9 @@ LevelOfDetail, ResolvedLevelOfDetail, ) -from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( - AudienceAudienceIdJobsGetJobIdParameter, +from rapidata.rapidata_client.benchmark._vote_filters import ( + demographic_filters, + in_filter, ) from rapidata.service.openapi_service import OpenAPIService from rapidata.api_client.models.update_leaderboard_endpoint_input import ( @@ -21,6 +22,8 @@ if TYPE_CHECKING: import pandas as pd from rapidata.rapidata_client.job.rapidata_job import RapidataJob + from rapidata.rapidata_client.filter.models.gender import Gender + from rapidata.rapidata_client.filter.models.age_group import AgeGroup class RapidataLeaderboard: @@ -224,23 +227,51 @@ def jobs(self) -> list[RapidataJob]: return [job_manager.get_job_by_id(job_id) for job_id in job_ids] - def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame": + def get_standings( + self, + tags: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, + ) -> "pd.DataFrame": """ Returns the standings of the leaderboard. + The demographic filters compute the standings from only the votes cast by + matching voters — e.g. the standings among women, or among US voters. + ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` and + ``language`` are observed. + Args: tags: The matchups with these tags should be used to create the standings. If tags are None, all matchups will be considered. If tags are empty, no matchups will be considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. Returns: A pandas DataFrame containing the standings of the leaderboard. """ with tracer.start_as_current_span("RapidataLeaderboard.get_standings"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) participants = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_standings_query_get( - leaderboard_id=self.id, tags=tags_filter + leaderboard_id=self.id, + tags=in_filter(tags), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) import pandas as pd @@ -266,6 +297,12 @@ def get_win_loss_matrix( self, tags: Optional[list[str]] = None, use_weighted_scoring: Optional[bool] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, ) -> pd.DataFrame: """ Returns the pairwise win/loss matrix for the participants in this leaderboard. @@ -277,6 +314,10 @@ def get_win_loss_matrix( always 0. This is the head-to-head breakdown behind :meth:`get_standings`, which collapses the same matchups into a single Elo score per model. + The demographic filters restrict the matrix to matchups decided by matching + voters. ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` + and ``language`` are observed. + Args: tags: Only count matchups carrying one of these prompt tags. If None, every matchup on the leaderboard is included; if an empty list, none are. @@ -285,6 +326,12 @@ def get_win_loss_matrix( plain win, so cells hold weighted sums (floats) rather than raw counts. If False, cells are raw win counts. When None (default), the server applies the leaderboard's configured default. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. Returns: A pandas DataFrame indexed by participant name on both axes, where cell @@ -292,12 +339,19 @@ def get_win_loss_matrix( participant over the column participant. """ with tracer.start_as_current_span("RapidataLeaderboard.get_win_loss_matrix"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) result = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_matrix_query_get( leaderboard_id=self.id, - tags=tags_filter, + tags=in_filter(tags), use_weighted_scoring=use_weighted_scoring, + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) import pandas as pd diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index e1e8729c8..2aff07082 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -11,8 +11,12 @@ BenchmarkPrompt, BenchmarkPromptUploader, ) -from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import ( - AudienceAudienceIdJobsGetJobIdParameter, +from rapidata.api_client.models.benchmark_demographic_dimension import ( + BenchmarkDemographicDimension, +) +from rapidata.rapidata_client.benchmark._vote_filters import ( + demographic_filters, + in_filter, ) if TYPE_CHECKING: @@ -24,6 +28,8 @@ from rapidata.rapidata_client.benchmark.participant.participant import ( BenchmarkParticipant, ) + from rapidata.rapidata_client.filter.models.gender import Gender + from rapidata.rapidata_client.filter.models.age_group import AgeGroup from rapidata.rapidata_client.settings import RapidataSetting from rapidata.service.openapi_service import OpenAPIService @@ -713,26 +719,47 @@ def get_overall_standings( self, tags: Optional[list[str]] = None, leaderboard_ids: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, ) -> pd.DataFrame: """ Returns an aggregated elo table of all leaderboards in the benchmark. + The demographic filters compute the standings from only the votes cast by + matching voters — e.g. the standings among women, or among US voters. + ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` and + ``language`` are observed. + Args: tags: Filter standings by these tags. If None, all tags are considered. leaderboard_ids: Filter to only include matchups from these leaderboards. If None, all leaderboards are considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. """ import pandas as pd with tracer.start_as_current_span("get_overall_standings"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags - leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter() - leaderboard_filter.var_in = leaderboard_ids - + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) participants = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_standings_query_get( benchmark_id=self.id, - tags=tags_filter, - leaderboard_id=leaderboard_filter, + tags=in_filter(tags), + leaderboard_id=in_filter(leaderboard_ids), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) standings = [] @@ -758,6 +785,12 @@ def get_win_loss_matrix( participant_ids: Optional[list[str]] = None, leaderboard_ids: Optional[list[str]] = None, use_weighted_scoring: Optional[bool] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, ) -> pd.DataFrame: """ Returns the pairwise win/loss matrix aggregated across the benchmark's leaderboards. @@ -770,6 +803,10 @@ def get_win_loss_matrix( head-to-head breakdown behind :meth:`get_overall_standings`, which collapses the same matchups into a single Elo score per model. + The demographic filters restrict the matrix to matchups decided by matching + voters. ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` + and ``language`` are observed. + Args: tags: Only count matchups carrying one of these prompt tags. If None, every matchup is included; if an empty list, none are. @@ -782,6 +819,12 @@ def get_win_loss_matrix( plain win, so cells hold weighted sums (floats) rather than raw counts. If False, cells are raw win counts. When None (default), the server applies its configured default. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. Returns: A pandas DataFrame indexed by participant name on both axes, where cell @@ -791,19 +834,21 @@ def get_win_loss_matrix( import pandas as pd with tracer.start_as_current_span("get_win_loss_matrix"): - tags_filter = AudienceAudienceIdJobsGetJobIdParameter() - tags_filter.var_in = tags - participant_filter = AudienceAudienceIdJobsGetJobIdParameter() - participant_filter.var_in = participant_ids - leaderboard_filter = AudienceAudienceIdJobsGetJobIdParameter() - leaderboard_filter.var_in = leaderboard_ids - + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_matrix_query_get( benchmark_id=self.id, - tags=tags_filter, - participant_id=participant_filter, - leaderboard_id=leaderboard_filter, + tags=in_filter(tags), + participant_id=in_filter(participant_ids), + leaderboard_id=in_filter(leaderboard_ids), use_weighted_scoring=use_weighted_scoring, + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, ) return pd.DataFrame( @@ -812,6 +857,172 @@ def get_win_loss_matrix( columns=pd.Index(result.columns), ) + def get_demographics( + self, + tags: Optional[list[str]] = None, + leaderboard_ids: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, + ) -> pd.DataFrame: + """ + Returns the demographic composition of the voters for this benchmark. + + One row per (dimension, bucket): the ``votes`` cast by that bucket and + its ``share`` of the dimension's votes (a dimension's shares sum to 1). + Every dimension (``AgeBucket``, ``Gender``, ``Occupation``, ``Country``, + ``Language``) includes an ``"unknown"`` bucket for votes whose attribute + could not be determined. + + ``AgeBucket``, ``Gender`` and ``Occupation`` are estimated (inferred from + behaviour), not self-declared; ``Country`` and ``Language`` are observed. + The demographic filters restrict the composition to matching voters. + + Args: + tags: Only count votes on matchups with these tags. If None, all matchups are considered. + leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. If None, all runs are considered. + + Returns: + A pandas DataFrame with columns ``dimension``, ``value``, ``votes``, ``share``. + """ + import pandas as pd + + with tracer.start_as_current_span("RapidataBenchmark.get_demographics"): + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) + result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_demographics_get( + benchmark_id=self.id, + tags=in_filter(tags), + leaderboard_id=in_filter(leaderboard_ids), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, + ) + + dimensions = result.dimensions + dimension_buckets = { + BenchmarkDemographicDimension.AGEBUCKET: dimensions.age_bucket, + BenchmarkDemographicDimension.GENDER: dimensions.gender, + BenchmarkDemographicDimension.OCCUPATION: dimensions.occupation, + BenchmarkDemographicDimension.COUNTRY: dimensions.country, + BenchmarkDemographicDimension.LANGUAGE: dimensions.language, + } + rows = [ + { + "dimension": dimension.value, + "value": bucket.value, + "votes": bucket.votes, + "share": bucket.share, + } + for dimension, buckets in dimension_buckets.items() + for bucket in buckets + ] + + return pd.DataFrame( + rows, columns=pd.Index(["dimension", "value", "votes", "share"]) + ) + + def get_standings_breakdown( + self, + dimension: BenchmarkDemographicDimension, + tags: Optional[list[str]] = None, + leaderboard_ids: Optional[list[str]] = None, + country: Optional[list[str]] = None, + language: Optional[list[str]] = None, + gender: Optional[list[Gender]] = None, + age_bucket: Optional[list[AgeGroup]] = None, + occupation: Optional[list[str]] = None, + run_id: Optional[str] = None, + ) -> pd.DataFrame: + """ + Returns the standings split by a demographic dimension of the voters. + + One row per (segment, model): how each demographic segment of voters + ranks the models, alongside that segment's vote count. The segments + include an ``"unknown"`` bucket. Scores are raw vote counts. For the + overall standings across all voters, use :py:meth:`get_overall_standings`. + + ``AgeBucket``, ``Gender`` and ``Occupation`` are estimated (inferred), not + self-declared; ``Country`` and ``Language`` are observed. The demographic + filters narrow the voters before the split (e.g. break down by age within + US voters only). + + Args: + dimension: The :class:`BenchmarkDemographicDimension` to split by (``AgeBucket``, ``Gender``, ``Occupation``, ``Country`` or ``Language``). + tags: Only count votes on matchups with these tags. If None, all matchups are considered. + leaderboard_ids: Only count votes from these leaderboards. If None, all leaderboards are considered. + country: Only count votes from these countries (ISO-2 codes). + language: Only count votes from these languages. + gender: Only count votes from voters of these (estimated) genders. + age_bucket: Only count votes from voters in these (estimated) age buckets. + occupation: Only count votes from voters of these (estimated) occupations. + run_id: Only count votes from this evaluation run. If None, all runs are considered. + + Returns: + A pandas DataFrame with columns ``segment``, ``segment_votes``, ``name``, ``wins``, ``total_matches``, ``score``. + """ + import pandas as pd + + with tracer.start_as_current_span("RapidataBenchmark.get_standings_breakdown"): + votes = demographic_filters( + country, language, gender, age_bucket, occupation, run_id + ) + result = self._openapi_service.leaderboard.benchmark_api.benchmark_benchmark_id_standings_breakdown_get( + benchmark_id=self.id, + dimension=BenchmarkDemographicDimension(dimension), + tags=in_filter(tags), + leaderboard_id=in_filter(leaderboard_ids), + country=votes.country, + language=votes.language, + gender=votes.gender, + age_bucket=votes.age_bucket, + occupation=votes.occupation, + run_id=votes.run_id, + ) + + rows = [] + for segment in result.segments: + for item in segment.items: + rows.append( + { + "segment": segment.value, + "segment_votes": segment.votes, + "name": item.name, + "wins": item.wins, + "total_matches": item.total_matches, + "score": ( + round(item.score, 2) if item.score is not None else None + ), + } + ) + + return pd.DataFrame( + rows, + columns=pd.Index( + [ + "segment", + "segment_votes", + "name", + "wins", + "total_matches", + "score", + ] + ), + ) + def __str__(self) -> str: return f"RapidataBenchmark(name={self.name}, id={self.id})"