From 04550f492674420a4206b29819c6157d48609e4e Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 17 Jul 2026 09:03:33 +0000 Subject: [PATCH 1/2] acc: run Lakebase invariant tests on AWS and Azure via CONFIG_CloudName Add a CONFIG_CloudName= pseudo-var alongside CONFIG_Cloud so EnvMatrixExclude rules can drop a variant on a single cloud rather than all clouds. It rides the same match-then-strip path in ExpandEnvMatrix, so it never leaks into the test environment, subtest names, or replacements. Use it to gate the Lakebase invariant templates on GCP only: both Lakebase v2 (postgres_*) and Lakebase v1 (database instances/catalogs/synced tables) are unavailable on GCP but work on AWS and Azure, so they now run on both instead of being excluded from all cloud runs. Co-authored-by: Isaac --- acceptance/acceptance_test.go | 3 +++ acceptance/bundle/invariant/test.toml | 27 ++++++++++++++------------- acceptance/internal/config.go | 1 + acceptance/internal/config_test.go | 27 +++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e5781f88f43..09f562ae013 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -534,6 +534,9 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { var extraVars []string if cloudEnv != "" { extraVars = append(extraVars, "CONFIG_Cloud=true") + // CONFIG_CloudName carries the normalized cloud (aws/azure/gcp) so + // EnvMatrixExclude rules can drop a variant on one cloud only. + extraVars = append(extraVars, "CONFIG_CloudName="+getCloudEnvBase(cloudEnv)) } expanded := internal.ExpandEnvMatrix(config.EnvMatrix, config.EnvMatrixExclude, extraVars) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index bd8f84aa6db..2ea942e4576 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -88,20 +88,21 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] # so this config is local-only (the mock server stores it verbatim). no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] -# Postgres resources only work on AWS -no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] -no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] -no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] -no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_catalog.yml.tmpl"] -no_postgres_synced_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"] -no_postgres_role_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_role.yml.tmpl"] -no_postgres_database_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_database.yml.tmpl"] +# Lakebase v2 (postgres_* resources) is not available on GCP, but works on AWS and Azure, +# so exclude these on GCP only. +no_postgres_project_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_project.yml.tmpl"] +no_postgres_branch_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_branch.yml.tmpl"] +no_postgres_endpoint_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +no_postgres_catalog_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_catalog.yml.tmpl"] +no_postgres_synced_table_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"] +no_postgres_role_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_role.yml.tmpl"] +no_postgres_database_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_database.yml.tmpl"] -# Lakebase v1 (database instances/catalogs/synced tables) is not available on GCP, -# so exclude these from all cloud runs (same as the postgres resources above). -no_database_instance_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=database_instance.yml.tmpl"] -no_database_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=database_catalog.yml.tmpl"] -no_synced_database_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=synced_database_table.yml.tmpl"] +# Lakebase v1 (database instances/catalogs/synced tables) is not available on GCP, but works on +# AWS and Azure, so exclude these on GCP only. +no_database_instance_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=database_instance.yml.tmpl"] +no_database_catalog_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=database_catalog.yml.tmpl"] +no_synced_database_table_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=synced_database_table.yml.tmpl"] # External locations require actual storage credentials with cloud IAM setup # which are environment-specific, so we only test locally with the mock server diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index b2d1a9f578f..3d8dd153098 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -118,6 +118,7 @@ type TestConfig struct { // For example: // EnvMatrixExclude.noplantf = ["READPLAN=1", "DATABRICKS_BUNDLE_ENGINE=terraform"] // Special pseudo-var CONFIG_Cloud=true is available for excluding combinations on cloud runs. + // Special pseudo-var CONFIG_CloudName= is available for excluding combinations on a specific cloud. EnvMatrixExclude map[string][]string // List of keys for which to do string replacement value -> [KEY]. If not set, defaults to true. diff --git a/acceptance/internal/config_test.go b/acceptance/internal/config_test.go index a81345ed571..87b0d88b960 100644 --- a/acceptance/internal/config_test.go +++ b/acceptance/internal/config_test.go @@ -193,6 +193,33 @@ func TestExpandEnvMatrix(t *testing.T) { {"KEY=B"}, }, }, + { + name: "CONFIG_CloudName excludes variant on matching cloud", + matrix: map[string][]string{ + "INPUT_CONFIG": {"database_instance.yml.tmpl", "job.yml.tmpl"}, + }, + exclude: map[string][]string{ + "no_database_instance_on_gcp": {"CONFIG_CloudName=gcp", "INPUT_CONFIG=database_instance.yml.tmpl"}, + }, + extraVars: []string{"CONFIG_Cloud=true", "CONFIG_CloudName=gcp"}, + expected: [][]string{ + {"INPUT_CONFIG=job.yml.tmpl"}, + }, + }, + { + name: "CONFIG_CloudName keeps variant on non-matching cloud", + matrix: map[string][]string{ + "INPUT_CONFIG": {"database_instance.yml.tmpl", "job.yml.tmpl"}, + }, + exclude: map[string][]string{ + "no_database_instance_on_gcp": {"CONFIG_CloudName=gcp", "INPUT_CONFIG=database_instance.yml.tmpl"}, + }, + extraVars: []string{"CONFIG_Cloud=true", "CONFIG_CloudName=aws"}, + expected: [][]string{ + {"INPUT_CONFIG=database_instance.yml.tmpl"}, + {"INPUT_CONFIG=job.yml.tmpl"}, + }, + }, } for _, tt := range tests { From fbd7435d40eba61871b4e0eb07247dad98634b70 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 17 Jul 2026 12:14:53 +0000 Subject: [PATCH 2/2] acc: fix postgres invariant templates for cloud deploy Running the invariant postgres templates on cloud (see the CONFIG_CloudName change) surfaced pre-existing template bugs that never fired because these templates had never run against a real backend: - postgres_endpoint declared an explicit read-write endpoint, colliding with the one the branch auto-provisions ("read_write endpoint already exists"). Take over the implicit production branch and primary endpoint with replace_existing. - postgres_branch created a new branch; switch it to take over the implicitly provisioned production branch with replace_existing for consistency. - postgres_synced_table referenced a project/branch it never declared ("Project not found") and a nonexistent source table. Declare the full base-resource chain (project, production branch, catalog, pipeline storage) and create a real source table via an init/cleanup script pair, mirroring the dedicated postgres_synced_tables resource test. replace_existing does not exist in v0.293.0, so the seed deploy in continue_293 drops the unknown field and then fails creating the already-provisioned branch. Exclude the three replace_existing templates from continue_293; they stay covered by no_drift and migrate. Verified green on AWS and Azure. Co-authored-by: Isaac --- .../configs/postgres_branch.yml.tmpl | 5 ++- .../configs/postgres_endpoint.yml.tmpl | 9 ++++- .../configs/postgres_synced_table.yml.tmpl | 39 ++++++++++++++++--- .../postgres_synced_table.yml.tmpl-cleanup.sh | 6 +++ .../postgres_synced_table.yml.tmpl-init.sh | 15 +++++++ .../invariant/continue_293/out.test.toml | 3 -- .../bundle/invariant/continue_293/test.toml | 7 ++++ 7 files changed, 72 insertions(+), 12 deletions(-) create mode 100755 acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-cleanup.sh create mode 100755 acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-init.sh diff --git a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl index e17859b0b90..03aac05eb94 100644 --- a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl @@ -7,8 +7,11 @@ resources: project_id: test-pg-project-$UNIQUE_NAME display_name: Test Postgres Project + # Take over the branch implicitly provisioned with the project rather than + # creating a new one. postgres_branches: foo: parent: ${resources.postgres_projects.project.name} - branch_id: test-branch-$UNIQUE_NAME + branch_id: production + replace_existing: true no_expiry: true diff --git a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl index fbffd428ff6..3a4a8b9f67f 100644 --- a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl @@ -7,14 +7,19 @@ resources: project_id: test-pg-project-$UNIQUE_NAME display_name: Test Postgres Project + # Creating the project implicitly provisions a "production" branch and its + # primary read-write endpoint, so take both over with replace_existing rather + # than creating new ones (a second read_write endpoint is rejected with 400). postgres_branches: branch: parent: ${resources.postgres_projects.project.name} - branch_id: test-branch-$UNIQUE_NAME + branch_id: production + replace_existing: true no_expiry: true postgres_endpoints: foo: parent: ${resources.postgres_branches.branch.name} - endpoint_id: test-endpoint-$UNIQUE_NAME + endpoint_id: primary + replace_existing: true endpoint_type: ENDPOINT_TYPE_READ_WRITE diff --git a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl index 7a548a02756..08c9a265769 100644 --- a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl @@ -2,15 +2,42 @@ bundle: name: test-bundle-$UNIQUE_NAME resources: + # Storage for the synced table's backing pipeline. + schemas: + pipeline_storage: + name: pipeline_storage_$UNIQUE_NAME + catalog_name: main + + postgres_projects: + project: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + + # Take over the branch implicitly provisioned with the project rather than + # creating a new one. + postgres_branches: + branch: + parent: ${resources.postgres_projects.project.name} + branch_id: production + replace_existing: true + no_expiry: true + + postgres_catalogs: + catalog: + catalog_id: test_pg_catalog_$UNIQUE_NAME + branch: ${resources.postgres_branches.branch.name} + postgres_database: appdb + create_database_if_missing: true + postgres_synced_tables: foo: - synced_table_id: lakebase_$UNIQUE_NAME.public.trips_synced - source_table_full_name: main.raw.trips - primary_key_columns: [id] + synced_table_id: ${resources.postgres_catalogs.catalog.catalog_id}.public.trips_synced + source_table_full_name: main.test_synced_$UNIQUE_NAME.trips_source + primary_key_columns: [tpep_pickup_datetime] scheduling_policy: SNAPSHOT postgres_database: appdb - branch: projects/test-pg-project-$UNIQUE_NAME/branches/production + branch: ${resources.postgres_branches.branch.name} create_database_objects_if_missing: true new_pipeline_spec: - storage_catalog: main - storage_schema: pipelines + storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} + storage_schema: ${resources.schemas.pipeline_storage.name} diff --git a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-cleanup.sh b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-cleanup.sh new file mode 100755 index 00000000000..4890ef14814 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Clean up the temporary source table +echo "Cleaning up temporary source table" +$CLI tables delete main.test_synced_$UNIQUE_NAME.trips_source || true +$CLI schemas delete main.test_synced_$UNIQUE_NAME || true diff --git a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-init.sh new file mode 100755 index 00000000000..4a47f8aa2bd --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-init.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Create a unique source table for this test run to avoid hitting the 20-table-per-source limit +echo "Creating temporary source table: main.test_synced_$UNIQUE_NAME.trips_source" + +# Create schema using CLI +$CLI schemas create test_synced_$UNIQUE_NAME main -o json | jq '{full_name}' + +# Create source table from samples.nyctaxi.trips using SQL API +# MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /api/... to C:/Program Files/Git/api/... +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.test_synced_$UNIQUE_NAME.trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10\", + \"wait_timeout\": \"45s\" + }" > /dev/null diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 8fd536c2b85..33b84295aa4 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -27,13 +27,10 @@ EnvMatrix.INPUT_CONFIG = [ "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", - "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", "postgres_database.yml.tmpl", - "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "postgres_role.yml.tmpl", - "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_uppercase_name.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 902316bea12..6aece425d10 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -26,3 +26,10 @@ EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min # per variant (two full deploys at 1000 tasks) without incremental coverage. EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] + +# replace_existing is not supported on v0.293.0: the seed deploy drops the unknown +# field and then fails trying to create the implicitly-provisioned production branch +# / primary endpoint. Covered by no_drift and migrate on current CLI. +EnvMatrixExclude.no_postgres_branch = ["INPUT_CONFIG=postgres_branch.yml.tmpl"] +EnvMatrixExclude.no_postgres_endpoint = ["INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +EnvMatrixExclude.no_postgres_synced_table = ["INPUT_CONFIG=postgres_synced_table.yml.tmpl"]