diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e5781f88f43..09f562ae013 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -534,6 +534,9 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { var extraVars []string if cloudEnv != "" { extraVars = append(extraVars, "CONFIG_Cloud=true") + // CONFIG_CloudName carries the normalized cloud (aws/azure/gcp) so + // EnvMatrixExclude rules can drop a variant on one cloud only. + extraVars = append(extraVars, "CONFIG_CloudName="+getCloudEnvBase(cloudEnv)) } expanded := internal.ExpandEnvMatrix(config.EnvMatrix, config.EnvMatrixExclude, extraVars) diff --git a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl index e17859b0b90..03aac05eb94 100644 --- a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl @@ -7,8 +7,11 @@ resources: project_id: test-pg-project-$UNIQUE_NAME display_name: Test Postgres Project + # Take over the branch implicitly provisioned with the project rather than + # creating a new one. postgres_branches: foo: parent: ${resources.postgres_projects.project.name} - branch_id: test-branch-$UNIQUE_NAME + branch_id: production + replace_existing: true no_expiry: true diff --git a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl index fbffd428ff6..3a4a8b9f67f 100644 --- a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl @@ -7,14 +7,19 @@ resources: project_id: test-pg-project-$UNIQUE_NAME display_name: Test Postgres Project + # Creating the project implicitly provisions a "production" branch and its + # primary read-write endpoint, so take both over with replace_existing rather + # than creating new ones (a second read_write endpoint is rejected with 400). postgres_branches: branch: parent: ${resources.postgres_projects.project.name} - branch_id: test-branch-$UNIQUE_NAME + branch_id: production + replace_existing: true no_expiry: true postgres_endpoints: foo: parent: ${resources.postgres_branches.branch.name} - endpoint_id: test-endpoint-$UNIQUE_NAME + endpoint_id: primary + replace_existing: true endpoint_type: ENDPOINT_TYPE_READ_WRITE diff --git a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl index 7a548a02756..08c9a265769 100644 --- a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl @@ -2,15 +2,42 @@ bundle: name: test-bundle-$UNIQUE_NAME resources: + # Storage for the synced table's backing pipeline. + schemas: + pipeline_storage: + name: pipeline_storage_$UNIQUE_NAME + catalog_name: main + + postgres_projects: + project: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + + # Take over the branch implicitly provisioned with the project rather than + # creating a new one. + postgres_branches: + branch: + parent: ${resources.postgres_projects.project.name} + branch_id: production + replace_existing: true + no_expiry: true + + postgres_catalogs: + catalog: + catalog_id: test_pg_catalog_$UNIQUE_NAME + branch: ${resources.postgres_branches.branch.name} + postgres_database: appdb + create_database_if_missing: true + postgres_synced_tables: foo: - synced_table_id: lakebase_$UNIQUE_NAME.public.trips_synced - source_table_full_name: main.raw.trips - primary_key_columns: [id] + synced_table_id: ${resources.postgres_catalogs.catalog.catalog_id}.public.trips_synced + source_table_full_name: main.test_synced_$UNIQUE_NAME.trips_source + primary_key_columns: [tpep_pickup_datetime] scheduling_policy: SNAPSHOT postgres_database: appdb - branch: projects/test-pg-project-$UNIQUE_NAME/branches/production + branch: ${resources.postgres_branches.branch.name} create_database_objects_if_missing: true new_pipeline_spec: - storage_catalog: main - storage_schema: pipelines + storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} + storage_schema: ${resources.schemas.pipeline_storage.name} diff --git a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-cleanup.sh b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-cleanup.sh new file mode 100755 index 00000000000..4890ef14814 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Clean up the temporary source table +echo "Cleaning up temporary source table" +$CLI tables delete main.test_synced_$UNIQUE_NAME.trips_source || true +$CLI schemas delete main.test_synced_$UNIQUE_NAME || true diff --git a/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-init.sh new file mode 100755 index 00000000000..4a47f8aa2bd --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_synced_table.yml.tmpl-init.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Create a unique source table for this test run to avoid hitting the 20-table-per-source limit +echo "Creating temporary source table: main.test_synced_$UNIQUE_NAME.trips_source" + +# Create schema using CLI +$CLI schemas create test_synced_$UNIQUE_NAME main -o json | jq '{full_name}' + +# Create source table from samples.nyctaxi.trips using SQL API +# MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /api/... to C:/Program Files/Git/api/... +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.test_synced_$UNIQUE_NAME.trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10\", + \"wait_timeout\": \"45s\" + }" > /dev/null diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 8fd536c2b85..33b84295aa4 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -27,13 +27,10 @@ EnvMatrix.INPUT_CONFIG = [ "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", - "postgres_branch.yml.tmpl", "postgres_catalog.yml.tmpl", "postgres_database.yml.tmpl", - "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "postgres_role.yml.tmpl", - "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_uppercase_name.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 902316bea12..6aece425d10 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -26,3 +26,10 @@ EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min # per variant (two full deploys at 1000 tasks) without incremental coverage. EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] + +# replace_existing is not supported on v0.293.0: the seed deploy drops the unknown +# field and then fails trying to create the implicitly-provisioned production branch +# / primary endpoint. Covered by no_drift and migrate on current CLI. +EnvMatrixExclude.no_postgres_branch = ["INPUT_CONFIG=postgres_branch.yml.tmpl"] +EnvMatrixExclude.no_postgres_endpoint = ["INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +EnvMatrixExclude.no_postgres_synced_table = ["INPUT_CONFIG=postgres_synced_table.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index bd8f84aa6db..2ea942e4576 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -88,20 +88,21 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] # so this config is local-only (the mock server stores it verbatim). no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] -# Postgres resources only work on AWS -no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] -no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] -no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] -no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_catalog.yml.tmpl"] -no_postgres_synced_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"] -no_postgres_role_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_role.yml.tmpl"] -no_postgres_database_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_database.yml.tmpl"] +# Lakebase v2 (postgres_* resources) is not available on GCP, but works on AWS and Azure, +# so exclude these on GCP only. +no_postgres_project_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_project.yml.tmpl"] +no_postgres_branch_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_branch.yml.tmpl"] +no_postgres_endpoint_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +no_postgres_catalog_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_catalog.yml.tmpl"] +no_postgres_synced_table_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_synced_table.yml.tmpl"] +no_postgres_role_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_role.yml.tmpl"] +no_postgres_database_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=postgres_database.yml.tmpl"] -# Lakebase v1 (database instances/catalogs/synced tables) is not available on GCP, -# so exclude these from all cloud runs (same as the postgres resources above). -no_database_instance_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=database_instance.yml.tmpl"] -no_database_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=database_catalog.yml.tmpl"] -no_synced_database_table_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=synced_database_table.yml.tmpl"] +# Lakebase v1 (database instances/catalogs/synced tables) is not available on GCP, but works on +# AWS and Azure, so exclude these on GCP only. +no_database_instance_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=database_instance.yml.tmpl"] +no_database_catalog_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=database_catalog.yml.tmpl"] +no_synced_database_table_on_gcp = ["CONFIG_CloudName=gcp", "INPUT_CONFIG=synced_database_table.yml.tmpl"] # External locations require actual storage credentials with cloud IAM setup # which are environment-specific, so we only test locally with the mock server diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index b2d1a9f578f..3d8dd153098 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -118,6 +118,7 @@ type TestConfig struct { // For example: // EnvMatrixExclude.noplantf = ["READPLAN=1", "DATABRICKS_BUNDLE_ENGINE=terraform"] // Special pseudo-var CONFIG_Cloud=true is available for excluding combinations on cloud runs. + // Special pseudo-var CONFIG_CloudName= is available for excluding combinations on a specific cloud. EnvMatrixExclude map[string][]string // List of keys for which to do string replacement value -> [KEY]. If not set, defaults to true. diff --git a/acceptance/internal/config_test.go b/acceptance/internal/config_test.go index a81345ed571..87b0d88b960 100644 --- a/acceptance/internal/config_test.go +++ b/acceptance/internal/config_test.go @@ -193,6 +193,33 @@ func TestExpandEnvMatrix(t *testing.T) { {"KEY=B"}, }, }, + { + name: "CONFIG_CloudName excludes variant on matching cloud", + matrix: map[string][]string{ + "INPUT_CONFIG": {"database_instance.yml.tmpl", "job.yml.tmpl"}, + }, + exclude: map[string][]string{ + "no_database_instance_on_gcp": {"CONFIG_CloudName=gcp", "INPUT_CONFIG=database_instance.yml.tmpl"}, + }, + extraVars: []string{"CONFIG_Cloud=true", "CONFIG_CloudName=gcp"}, + expected: [][]string{ + {"INPUT_CONFIG=job.yml.tmpl"}, + }, + }, + { + name: "CONFIG_CloudName keeps variant on non-matching cloud", + matrix: map[string][]string{ + "INPUT_CONFIG": {"database_instance.yml.tmpl", "job.yml.tmpl"}, + }, + exclude: map[string][]string{ + "no_database_instance_on_gcp": {"CONFIG_CloudName=gcp", "INPUT_CONFIG=database_instance.yml.tmpl"}, + }, + extraVars: []string{"CONFIG_Cloud=true", "CONFIG_CloudName=aws"}, + expected: [][]string{ + {"INPUT_CONFIG=database_instance.yml.tmpl"}, + {"INPUT_CONFIG=job.yml.tmpl"}, + }, + }, } for _, tt := range tests {