Skip to content

Allow configuring the collation of asset name columns #71249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
1fanwang wants to merge 1 commit into
base: main
Choose a base branch
from
+40 −92
Open

Allow configuring the collation of asset name columns #71249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions airflow-core/newsfragments/71249.improvement.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The collation used for the asset ``name``, ``uri`` and ``group`` columns on MySQL is now configurable through ``[database] sql_engine_collation_for_asset_names``. It still defaults to ``latin1_general_cs``, so existing databases are unaffected. Set it if your MySQL-compatible engine does not provide that collation.
11 changes: 11 additions & 0 deletions airflow-core/src/airflow/config_templates/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,17 @@ database:
type: string
example: ~
default: ~
sql_engine_collation_for_asset_names:
description: |
Collation for the ``name``, ``uri`` and ``group`` columns of the asset tables on
``mysql`` and ``mariadb``. These columns hold ASCII values and are indexed at 1500
characters, so a single-byte charset is used to stay within the maximum index size.
Override this if your database engine does not provide ``latin1_general_cs`` --
for example TiDB, which supports ``latin1_bin`` instead.
version_added: 3.4.0
type: string
example: "latin1_bin"
default: "latin1_general_cs"
sql_alchemy_pool_enabled:
description: |
If SQLAlchemy should pool database connections.
Expand Down
103 changes: 11 additions & 92 deletions airflow-core/src/airflow/models/asset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
Index,
Integer,
PrimaryKeyConstraint,
String,
Table,
delete,
select,
Expand All @@ -39,7 +38,7 @@
from sqlalchemy.orm import Mapped, mapped_column, relationship

from airflow._shared.timezones import timezone
from airflow.models.base import Base, StringID
from airflow.models.base import ASSET_STR_FIELD, Base, StringID
from airflow.utils.sqlalchemy import UtcDateTime

if TYPE_CHECKING:
Expand Down Expand Up @@ -147,15 +146,7 @@ class AssetWatcherModel(Base):
"""A table to store asset watchers."""

name: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
nullable=False,
)
asset_id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
Expand Down Expand Up @@ -198,27 +189,11 @@ class AssetAliasModel(Base):

id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
name: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
nullable=False,
)
group: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
default="",
nullable=False,
)
Expand Down Expand Up @@ -279,39 +254,15 @@ class AssetModel(Base):

id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
name: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
nullable=False,
)
uri: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
nullable=False,
)
group: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
default=str,
nullable=False,
)
Expand Down Expand Up @@ -408,27 +359,11 @@ class AssetActive(Base):
"""

name: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
nullable=False,
)
uri: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
nullable=False,
)

Expand Down Expand Up @@ -456,15 +391,7 @@ class DagScheduleAssetNameReference(Base):
"""Reference from a DAG to an asset name reference of which it is a consumer."""

name: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
primary_key=True,
nullable=False,
)
Expand Down Expand Up @@ -502,15 +429,7 @@ class DagScheduleAssetUriReference(Base):
"""Reference from a DAG to an asset URI reference of which it is a consumer."""

uri: Mapped[str] = mapped_column(
String(length=1500).with_variant(
String(
length=1500,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation="latin1_general_cs",
),
"mysql",
),
ASSET_STR_FIELD,
primary_key=True,
nullable=False,
)
Expand Down
17 changes: 17 additions & 0 deletions airflow-core/src/airflow/models/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,23 @@ def get_id_collation_args():
COLLATION_ARGS: dict[str, Any] = get_id_collation_args()


def get_asset_str_field(length: int = 1500) -> String:
"""
Build the string type used for asset name/uri/group columns.

On MySQL these carry an explicit latin1 collation: the values are ASCII, and
a 1-byte-per-character charset keeps the 1500-char unique indexes inside the
3072-byte index limit that utf8mb4 would blow past. The collation is
overridable because MySQL-compatible engines do not all ship
``latin1_general_cs`` (TiDB, for one, accepts only ``latin1_bin``).
"""
collation = conf.get("database", "sql_engine_collation_for_asset_names", fallback="latin1_general_cs")
return String(length=length).with_variant(String(length=length, collation=collation), "mysql")


ASSET_STR_FIELD: String = get_asset_str_field()


def StringID(*, length=ID_LEN, **kwargs) -> String:
return String(length=length, **kwargs, **COLLATION_ARGS)

Expand Down