From 4fa92d77851a0a5c99fb0ac255cae0416d48474e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 27 Aug 2026 11:57:44 +0200 Subject: [PATCH 1/2] acc: drop backend_default plan entries in UC comment tests catalogs/comment_out_of_band and schemas/comment_out_of_band dump the plan change map via `bundle plan --output json | jq '.plan[].changes'`. On GCP the backend populates unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged on the fresh catalog/schema, which #5877's wildcard classifies as a backend default; it surfaces as an extra reason:backend_default skip in the plan. AWS and Azure do not populate it, so the committed golden diverged and the nightly failed on GCP only. Teach normalize_uc_payload.py to drop any reason:backend_default change entry (is_backend_default), generalizing the managed-defaults handling to the shape jq leaves once it strips the enclosing changes key, and pipe both tests through it so all clouds collapse to the real change. Local output is unchanged: the testserver only injects managed props for the fixed catalog_managed_defaults name, not the unique names these cloud tests use. Co-authored-by: Isaac --- acceptance/bin/normalize_uc_payload.py | 28 +++++++++++++++++++ .../catalogs/comment_out_of_band/script | 2 +- .../schemas/comment_out_of_band/script | 2 +- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/acceptance/bin/normalize_uc_payload.py b/acceptance/bin/normalize_uc_payload.py index 943e73e1ed2..b62f4cb7fd2 100755 --- a/acceptance/bin/normalize_uc_payload.py +++ b/acceptance/bin/normalize_uc_payload.py @@ -14,6 +14,12 @@ an emptied `properties` object is then dropped so {} vs absent doesn't diverge. - a `changes` entry keyed `properties` or `properties['unity.catalog.managed...']`; an emptied `changes` object is likewise dropped. + - a whole plan change entry marked `reason: backend_default`, which `jq '.plan[].changes'` + extracts without its enclosing `changes` key; the entry is dropped outright. + +A backend default is a field the CLI never acts on but the backend populated, so it appears +only on some clouds — exactly the managed-defaults problem, generalized to any such field. +Dropping the entry (not just its managed keys) keeps goldens identical across clouds. Any field names passed as arguments are additionally deleted wherever they appear. These are the volatile server-set fields (created_at, metastore_id, schema_id, ...) each test @@ -30,6 +36,24 @@ managed_re = re.compile(r"unity\.catalog\.managed\..*\.defaults\..*") +def is_backend_default(value): + """Report whether a plan change entry is one the CLI skipped as a backend default. + + These are output-only (the CLI never acts on them) and only appear on clouds whose + backend populated the field, so the whole entry is dropped — this also catches the + managed-property change once `jq '.plan[].changes'` has stripped the `changes` key + that would otherwise route it through is_managed_change. + + >>> is_backend_default({"action": "skip", "reason": "backend_default", "remote": {"x": "1"}}) + True + >>> is_backend_default({"action": "update"}) + False + >>> is_backend_default("backend_default") + False + """ + return isinstance(value, dict) and value.get("reason") == "backend_default" + + def is_managed_change(key, value): """Report whether a plan `changes` entry is purely a managed-default change. @@ -66,6 +90,8 @@ def prune(node, drop_fields): {'properties': {'k': 'v'}} >>> prune({"changes": {"properties": {"action": "skip", "remote": {"unity.catalog.managed.a.defaults.b": "1"}}, "name": {"action": "update"}}}, set()) {'changes': {'name': {'action': 'update'}}} + >>> prune({"properties": {"action": "skip", "reason": "backend_default", "remote": {"k": "v"}}}, set()) + {} >>> prune([{"metastore_id": "m", "full_name": "c.s"}], {"metastore_id"}) [{'full_name': 'c.s'}] """ @@ -78,6 +104,8 @@ def prune(node, drop_fields): for key, value in node.items(): if key in drop_fields or managed_re.search(key): continue + if is_backend_default(value): + continue if key == "changes" and isinstance(value, dict): value = {k: v for k, v in value.items() if not is_managed_change(k, v)} pruned = prune(value, drop_fields) diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/script b/acceptance/bundle/resources/catalogs/comment_out_of_band/script index fe406882746..a8773fdc1eb 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/catalogs/$CATALOG" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the catalog" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | jq '.plan[].changes' | normalize_uc_payload.py title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/script b/acceptance/bundle/resources/schemas/comment_out_of_band/script index 70871c43a9b..74e47c2113a 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/script +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/schemas/$SCHEMA" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the schema" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | jq '.plan[].changes' | normalize_uc_payload.py title "Redeploy clears the comment" trace $CLI bundle deploy From 1793972456f7ffde32a7b5131bf2e19f34c48111 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 27 Aug 2026 13:00:32 +0200 Subject: [PATCH 2/2] acc: narrow normalize_uc_payload to managed-default changes only The first cut dropped every reason:backend_default change entry, which also stripped the deterministic storage_location backend default that volumes tests (volumes/uppercase-name, grants/volumes) show in their goldens, breaking local direct-engine acceptance. Only the managed-property change is cloud-divergent. Reuse is_managed_change and apply it at every level instead of only under a `changes` key, so it also fires on the top-level entry that jq '.plan[].changes' leaves behind. Plain backend defaults like storage_location are kept. Co-authored-by: Isaac --- acceptance/bin/normalize_uc_payload.py | 45 ++++++++------------------ 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/acceptance/bin/normalize_uc_payload.py b/acceptance/bin/normalize_uc_payload.py index b62f4cb7fd2..583e50a9f0c 100755 --- a/acceptance/bin/normalize_uc_payload.py +++ b/acceptance/bin/normalize_uc_payload.py @@ -9,17 +9,16 @@ yes, Azure no), so a committed golden cannot show them. The CLI already classifies them as backend defaults, so they never affect the plan; dropping them here is output-only. -Managed properties surface in three shapes, all handled by pruning matching keys: +Managed properties surface in these shapes, all handled by pruning matching keys: - a `properties` map (remote_state.properties, new_state.value.properties, ...); an emptied `properties` object is then dropped so {} vs absent doesn't diverge. - - a `changes` entry keyed `properties` or `properties['unity.catalog.managed...']`; - an emptied `changes` object is likewise dropped. - - a whole plan change entry marked `reason: backend_default`, which `jq '.plan[].changes'` - extracts without its enclosing `changes` key; the entry is dropped outright. + - a plan change entry that is purely a managed-default change (keyed `properties` with a + managed-only `remote`, or `properties['unity.catalog.managed...']`). It is dropped + wherever it appears — including at the top level, where `jq '.plan[].changes'` extracts + it without its enclosing `changes` key. An emptied `changes` object is likewise dropped. -A backend default is a field the CLI never acts on but the backend populated, so it appears -only on some clouds — exactly the managed-defaults problem, generalized to any such field. -Dropping the entry (not just its managed keys) keeps goldens identical across clouds. +Only the managed-default change is dropped, not every `backend_default` skip: a plain +backend default like a volume's `storage_location` is deterministic and stays in the golden. Any field names passed as arguments are additionally deleted wherever they appear. These are the volatile server-set fields (created_at, metastore_id, schema_id, ...) each test @@ -36,24 +35,6 @@ managed_re = re.compile(r"unity\.catalog\.managed\..*\.defaults\..*") -def is_backend_default(value): - """Report whether a plan change entry is one the CLI skipped as a backend default. - - These are output-only (the CLI never acts on them) and only appear on clouds whose - backend populated the field, so the whole entry is dropped — this also catches the - managed-property change once `jq '.plan[].changes'` has stripped the `changes` key - that would otherwise route it through is_managed_change. - - >>> is_backend_default({"action": "skip", "reason": "backend_default", "remote": {"x": "1"}}) - True - >>> is_backend_default({"action": "update"}) - False - >>> is_backend_default("backend_default") - False - """ - return isinstance(value, dict) and value.get("reason") == "backend_default" - - def is_managed_change(key, value): """Report whether a plan `changes` entry is purely a managed-default change. @@ -90,8 +71,10 @@ def prune(node, drop_fields): {'properties': {'k': 'v'}} >>> prune({"changes": {"properties": {"action": "skip", "remote": {"unity.catalog.managed.a.defaults.b": "1"}}, "name": {"action": "update"}}}, set()) {'changes': {'name': {'action': 'update'}}} - >>> prune({"properties": {"action": "skip", "reason": "backend_default", "remote": {"k": "v"}}}, set()) - {} + >>> prune({"comment": {"action": "update"}, "properties": {"action": "skip", "reason": "backend_default", "remote": {"unity.catalog.managed.a.defaults.b": "1"}}}, set()) + {'comment': {'action': 'update'}} + >>> prune({"storage_location": {"action": "skip", "reason": "backend_default"}}, set()) + {'storage_location': {'action': 'skip', 'reason': 'backend_default'}} >>> prune([{"metastore_id": "m", "full_name": "c.s"}], {"metastore_id"}) [{'full_name': 'c.s'}] """ @@ -104,10 +87,10 @@ def prune(node, drop_fields): for key, value in node.items(): if key in drop_fields or managed_re.search(key): continue - if is_backend_default(value): + # Drop a managed-default change wherever it sits: nested under `changes`, or at the + # top level after `jq '.plan[].changes'` has stripped the enclosing `changes` key. + if is_managed_change(key, value): continue - if key == "changes" and isinstance(value, dict): - value = {k: v for k, v in value.items() if not is_managed_change(k, v)} pruned = prune(value, drop_fields) # Drop a properties/changes object emptied by managed-key removal so goldens # don't diverge on {} (cloud that injects) vs absent (cloud that doesn't).