Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions airflow-core/src/airflow/api_fastapi/common/db/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from typing import TYPE_CHECKING, Annotated, Literal, overload

from fastapi import Depends
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session

Expand All @@ -37,6 +38,11 @@

from airflow.api_fastapi.core_api.base import OrmClause

# Rows a single scan reads. Result sets that fit are counted exactly; wider ones report a floor.
# Shared by the dashboard's historical metrics and by cursor-paginated listings, which surface an
# item count without counting every matching row — the point of cursor pagination on large tables.
EXACT_COUNT_LIMIT = 50_000


def _get_session() -> Generator[Session, None, None]:
with create_session(scoped=False) as session:
Expand All @@ -59,6 +65,30 @@ def apply_filters_to_select(
return statement


def bounded_total_entries(
*,
statement: Select,
filters: Sequence[OrmClause | None] | None = None,
session: Session,
) -> tuple[int, int]:
"""
Count the rows a cursor-paginated listing matches, reading at most ``EXACT_COUNT_LIMIT``.

Returns ``(total, limit)`` where ``total`` is ``min(actual_count, EXACT_COUNT_LIMIT)`` — a
``total`` equal to ``limit`` means only that at least that many rows match — and ``limit`` is
the cap that was applied, for the caller to surface as ``total_entries_limit``.

The ``LIMIT`` sits inside the counted subquery so the database stops scanning once the cap is
reached, keeping the count cheap on tables that cursor pagination exists to handle. ORDER BY is
stripped for the same reason :func:`~airflow.utils.db.get_query_count` strips it: it cannot
change a count and only constrains the planner.
"""
statement = apply_filters_to_select(statement=statement, filters=filters)
bounded = statement.order_by(None).limit(EXACT_COUNT_LIMIT).subquery()
total = session.scalar(select(func.count()).select_from(bounded)) or 0
return total, EXACT_COUNT_LIMIT


async def _get_async_session() -> AsyncGenerator[AsyncSession, None]:
async with create_session_async() as session:
yield session
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,14 @@ class DAGRunCollectionResponse(BaseModel):
dag_runs: Iterable[DAGRunResponse]
total_entries: int | None = Field(
default=None,
description="Total number of matching items. Populated for offset pagination, "
"``null`` when using cursor pagination.",
description="Number of matching items. For offset pagination this is the exact total. "
"For cursor pagination it is capped at ``total_entries_limit``; a value equal to that "
"limit means at least that many items match.",
)
total_entries_limit: int | None = Field(
default=None,
description="Cap applied to ``total_entries`` under cursor pagination. ``null`` for offset "
"pagination, where ``total_entries`` is exact.",
)
next_cursor: str | None = Field(
default=None,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,14 @@ class TaskInstanceCollectionResponse(BaseModel):
task_instances: Iterable[TaskInstanceResponse]
total_entries: int | None = Field(
default=None,
description="Total number of matching items. Populated for offset pagination, "
"``null`` when using cursor pagination.",
description="Number of matching items. For offset pagination this is the exact total. "
"For cursor pagination it is capped at ``total_entries_limit``; a value equal to that "
"limit means at least that many items match.",
)
total_entries_limit: int | None = Field(
default=None,
description="Cap applied to ``total_entries`` under cursor pagination. ``null`` for offset "
"pagination, where ``total_entries`` is exact.",
)
next_cursor: str | None = Field(
default=None,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2270,8 +2270,11 @@ paths:
**Cursor:** pass `cursor` (empty string for the first page, then `next_cursor`
from the response).

When `cursor` is provided, `offset` is ignored and `total_entries` is not
returned.
When `cursor` is provided, `offset` is ignored and `total_entries` is capped
at

`total_entries_limit` (a value equal to that limit means at least that many
runs match).

``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor``
is ``null``
Expand Down Expand Up @@ -8181,13 +8184,16 @@ paths:
**Cursor:** pass `cursor` (empty string for the first page, then `next_cursor`
from the response).

When `cursor` is provided, `offset` is ignored and `total_entries` is not
returned.
When `cursor` is provided, `offset` is ignored and `total_entries` is capped
at

``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor``
is ``null``
`total_entries_limit` (a value equal to that limit means at least that many
task instances

on the first page.'
match). ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor``
is

``null`` on the first page.'
operationId: get_task_instances
security:
- OAuth2PasswordBearer: []
Expand Down Expand Up @@ -13411,8 +13417,16 @@ components:
- type: integer
- type: 'null'
title: Total Entries
description: Total number of matching items. Populated for offset pagination,
``null`` when using cursor pagination.
description: Number of matching items. For offset pagination this is the
exact total. For cursor pagination it is capped at ``total_entries_limit``;
a value equal to that limit means at least that many items match.
total_entries_limit:
anyOf:
- type: integer
- type: 'null'
title: Total Entries Limit
description: Cap applied to ``total_entries`` under cursor pagination. ``null``
for offset pagination, where ``total_entries`` is exact.
next_cursor:
anyOf:
- type: string
Expand Down Expand Up @@ -15423,8 +15437,16 @@ components:
- type: integer
- type: 'null'
title: Total Entries
description: Total number of matching items. Populated for offset pagination,
``null`` when using cursor pagination.
description: Number of matching items. For offset pagination this is the
exact total. For cursor pagination it is capped at ``total_entries_limit``;
a value equal to that limit means at least that many items match.
total_entries_limit:
anyOf:
- type: integer
- type: 'null'
title: Total Entries Limit
description: Cap applied to ``total_entries`` under cursor pagination. ``null``
for offset pagination, where ``total_entries`` is exact.
next_cursor:
anyOf:
- type: string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,12 @@
parse_cursor,
)
from airflow.api_fastapi.common.dagbag import DagBagDep, get_dag_for_run, get_latest_version_of_dag
from airflow.api_fastapi.common.db.common import SessionDep, apply_filters_to_select, paginated_select
from airflow.api_fastapi.common.db.common import (
SessionDep,
apply_filters_to_select,
bounded_total_entries,
paginated_select,
)
from airflow.api_fastapi.common.db.dag_runs import (
attach_dag_versions_to_runs,
eager_load_dag_run_for_list,
Expand Down Expand Up @@ -585,7 +590,8 @@ def get_dag_runs(
**Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.

**Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
`total_entries_limit` (a value equal to that limit means at least that many runs match).
``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
on the first page.
"""
Expand Down Expand Up @@ -695,8 +701,13 @@ def get_dag_runs(
attach_dag_versions_to_runs(dag_runs, session=session)
attach_team_names(dag_runs, session=session)

total_entries, total_entries_limit = bounded_total_entries(
statement=query, filters=filters, session=session
)
return DAGRunCollectionResponse(
dag_runs=dag_runs,
total_entries=total_entries,
total_entries_limit=total_entries_limit,
next_cursor=(encode_cursor(dag_runs[-1], order_by) if has_next and dag_runs else None),
previous_cursor=(
make_backward_cursor(encode_cursor(dag_runs[0], order_by)) if has_prev and dag_runs else None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,12 @@
get_latest_version_of_dag,
resolve_run_on_latest_version,
)
from airflow.api_fastapi.common.db.common import SessionDep, apply_filters_to_select, paginated_select
from airflow.api_fastapi.common.db.common import (
SessionDep,
apply_filters_to_select,
bounded_total_entries,
paginated_select,
)
from airflow.api_fastapi.common.db.dags import attach_team_names
from airflow.api_fastapi.common.db.task_instances import eager_load_TI_and_TIH_for_validation
from airflow.api_fastapi.common.parameters import (
Expand Down Expand Up @@ -545,9 +550,10 @@ def get_task_instances(
**Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.

**Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
on the first page.
When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
`total_entries_limit` (a value equal to that limit means at least that many task instances
match). ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is
``null`` on the first page.
"""
use_cursor = cursor is not None
dag_run = None
Expand Down Expand Up @@ -642,8 +648,13 @@ def get_task_instances(

attach_team_names(task_instances, session=session)

total_entries, total_entries_limit = bounded_total_entries(
statement=query, filters=filters, session=session
)
return TaskInstanceCollectionResponse(
task_instances=task_instances,
total_entries=total_entries,
total_entries_limit=total_entries_limit,
next_cursor=(
encode_cursor(task_instances[-1], order_by) if has_next and task_instances else None
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep
from airflow.api_fastapi.common.db.common import EXACT_COUNT_LIMIT, SessionDep
from airflow.api_fastapi.common.parameters import DateTimeQuery, OptionalDateTimeQuery
from airflow.api_fastapi.common.router import AirflowRouter
from airflow.api_fastapi.core_api.datamodels.ui.dashboard import (
Expand All @@ -44,9 +44,6 @@

dashboard_router = AirflowRouter(tags=["Dashboard"], prefix="/dashboard")

# Rows a single scan reads. Windows that fit are counted exactly; wider ones report a floor.
EXACT_COUNT_LIMIT = 50_000


_ROUNDING = Context(prec=2, rounding=ROUND_FLOOR)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,8 @@ export const ensureUseDagRunServiceGetDagRunData = (queryClient: QueryClient, {
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many runs match).
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* @param data The data for the request.
Expand Down Expand Up @@ -1105,9 +1106,10 @@ export const ensureUseTaskInstanceServiceGetMappedTaskInstanceData = (queryClien
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many task instances
* match). ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is
* ``null`` on the first page.
* @param data The data for the request.
* @param data.dagId
* @param data.dagRunId
Expand Down
10 changes: 6 additions & 4 deletions airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,8 @@ export const prefetchUseDagRunServiceGetDagRun = (queryClient: QueryClient, { da
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many runs match).
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* @param data The data for the request.
Expand Down Expand Up @@ -1105,9 +1106,10 @@ export const prefetchUseTaskInstanceServiceGetMappedTaskInstance = (queryClient:
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many task instances
* match). ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is
* ``null`` on the first page.
* @param data The data for the request.
* @param data.dagId
* @param data.dagRunId
Expand Down
10 changes: 6 additions & 4 deletions airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,8 @@ export const useDagRunServiceGetDagRun = <TData = Common.DagRunServiceGetDagRunD
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many runs match).
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* @param data The data for the request.
Expand Down Expand Up @@ -1105,9 +1106,10 @@ export const useTaskInstanceServiceGetMappedTaskInstance = <TData = Common.TaskI
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many task instances
* match). ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is
* ``null`` on the first page.
* @param data The data for the request.
* @param data.dagId
* @param data.dagRunId
Expand Down
10 changes: 6 additions & 4 deletions airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,8 @@ export const useDagRunServiceGetDagRunSuspense = <TData = Common.DagRunServiceGe
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many runs match).
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* @param data The data for the request.
Expand Down Expand Up @@ -1105,9 +1106,10 @@ export const useTaskInstanceServiceGetMappedTaskInstanceSuspense = <TData = Comm
* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`.
*
* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response).
* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned.
* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null``
* on the first page.
* When `cursor` is provided, `offset` is ignored and `total_entries` is capped at
* `total_entries_limit` (a value equal to that limit means at least that many task instances
* match). ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is
* ``null`` on the first page.
* @param data The data for the request.
* @param data.dagId
* @param data.dagRunId
Expand Down
28 changes: 26 additions & 2 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3629,7 +3629,19 @@ export const $DAGRunCollectionResponse = {
}
],
title: 'Total Entries',
description: 'Total number of matching items. Populated for offset pagination, ``null`` when using cursor pagination.'
description: 'Number of matching items. For offset pagination this is the exact total. For cursor pagination it is capped at ``total_entries_limit``; a value equal to that limit means at least that many items match.'
},
total_entries_limit: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Total Entries Limit',
description: 'Cap applied to ``total_entries`` under cursor pagination. ``null`` for offset pagination, where ``total_entries`` is exact.'
},
next_cursor: {
anyOf: [
Expand Down Expand Up @@ -6576,7 +6588,19 @@ export const $TaskInstanceCollectionResponse = {
}
],
title: 'Total Entries',
description: 'Total number of matching items. Populated for offset pagination, ``null`` when using cursor pagination.'
description: 'Number of matching items. For offset pagination this is the exact total. For cursor pagination it is capped at ``total_entries_limit``; a value equal to that limit means at least that many items match.'
},
total_entries_limit: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Total Entries Limit',
description: 'Cap applied to ``total_entries`` under cursor pagination. ``null`` for offset pagination, where ``total_entries`` is exact.'
},
next_cursor: {
anyOf: [
Expand Down
Loading
Loading