From 16953f4f3e23563d61d23e67cd529967642b0f45 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Fri, 28 Aug 2026 10:36:47 +0200 Subject: [PATCH] Bump dbt-sql to DBR 16.4 LTS for classic compute Follow-up from #6378: - dbt-sql: bump latest_lts_dbr_version 15.4 -> 16.4 LTS for classic (non-serverless) compute, matching the default template's 16.4 pin and aligning the cluster runtime's Python version (3.12) with serverless environment version 5. - Remove the unused, TODO-flagged latest_lts_db_connect_version_spec macro from dbt-sql (flagged for removal in #6378; no references). - Add a dbt-sql-classic acceptance variant exercising the classic new_cluster path, locking in the bumped spark_version. Co-authored-by: Isaac --- .../bundles/dbt-sql-classic-dbr-16-4.md | 1 + .../templates/dbt-sql-classic/input.json | 7 +++ .../templates/dbt-sql-classic/out.test.toml | 2 + .../templates/dbt-sql-classic/output.txt | 45 +++++++++++++++++++ .../bundle/templates/dbt-sql-classic/script | 7 +++ .../templates/dbt-sql/library/versions.tmpl | 7 +-- 6 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 .nextchanges/bundles/dbt-sql-classic-dbr-16-4.md create mode 100644 acceptance/bundle/templates/dbt-sql-classic/input.json create mode 100644 acceptance/bundle/templates/dbt-sql-classic/out.test.toml create mode 100644 acceptance/bundle/templates/dbt-sql-classic/output.txt create mode 100644 acceptance/bundle/templates/dbt-sql-classic/script diff --git a/.nextchanges/bundles/dbt-sql-classic-dbr-16-4.md b/.nextchanges/bundles/dbt-sql-classic-dbr-16-4.md new file mode 100644 index 00000000000..1945933c3bc --- /dev/null +++ b/.nextchanges/bundles/dbt-sql-classic-dbr-16-4.md @@ -0,0 +1 @@ +The `dbt-sql` bundle template now uses Databricks Runtime 16.4 LTS (up from 15.4 LTS) for classic (non-serverless) compute. diff --git a/acceptance/bundle/templates/dbt-sql-classic/input.json b/acceptance/bundle/templates/dbt-sql-classic/input.json new file mode 100644 index 00000000000..51090158dda --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-classic/input.json @@ -0,0 +1,7 @@ +{ + "project_name": "my_dbt_sql", + "http_path": "/sql/2.0/warehouses/f00dcafe", + "default_catalog": "main", + "personal_schemas": "yes", + "serverless": "no" +} diff --git a/acceptance/bundle/templates/dbt-sql-classic/out.test.toml b/acceptance/bundle/templates/dbt-sql-classic/out.test.toml new file mode 100644 index 00000000000..98ea5040486 --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-classic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/dbt-sql-classic/output.txt b/acceptance/bundle/templates/dbt-sql-classic/output.txt new file mode 100644 index 00000000000..66c5cf492e0 --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-classic/output.txt @@ -0,0 +1,45 @@ + +>>> [CLI] bundle init dbt-sql --config-file ./input.json --output-dir output + +Welcome to the dbt template for Declarative Automation Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: [DATABRICKS_URL] + +📊 Your new project has been created in the 'my_dbt_sql' directory! +If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started. +Refer to the README.md file for full "getting started" guide and production setup instructions. + + +>>> diff.py [TESTROOT]/bundle/templates/dbt-sql-classic/../dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml output/my_dbt_sql/resources/my_dbt_sql.job.yml +--- [TESTROOT]/bundle/templates/dbt-sql-classic/../dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml ++++ output/my_dbt_sql/resources/my_dbt_sql.job.yml +@@ -16,5 +16,4 @@ + tasks: + - task_key: dbt +- environment_key: default + dbt_task: + project_directory: ../ +@@ -27,8 +26,16 @@ + - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + +- environments: +- - environment_key: default +- spec: +- environment_version: "5" +- dependencies: +- - dbt-databricks>=1.8.0,<2.0.0 ++ libraries: ++ - pypi: ++ package: dbt-databricks>=1.8.0,<2.0.0 ++ ++ new_cluster: ++ spark_version: 16.4.x-scala2.12 ++ node_type_id: [NODE_TYPE_ID] ++ data_security_mode: DATA_SECURITY_MODE_AUTO ++ num_workers: 0 ++ spark_conf: ++ spark.master: "local[*, 4]" ++ spark.databricks.cluster.profile: singleNode ++ custom_tags: ++ ResourceClass: SingleNode diff --git a/acceptance/bundle/templates/dbt-sql-classic/script b/acceptance/bundle/templates/dbt-sql-classic/script new file mode 100644 index 00000000000..a9c1fb43ffc --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-classic/script @@ -0,0 +1,7 @@ +# With serverless: no, the dbt-sql job runs on a classic new_cluster instead of a +# serverless environment. Only the job resource differs from the serverless dbt-sql +# variant, so we diff just that file against the dbt-sql golden. This locks in the +# new_cluster spark_version (latest_lts_dbr_version). +trace $CLI bundle init dbt-sql --config-file ./input.json --output-dir output +trace diff.py $TESTDIR/../dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml output/my_dbt_sql/resources/my_dbt_sql.job.yml +rm -fr output diff --git a/libs/template/templates/dbt-sql/library/versions.tmpl b/libs/template/templates/dbt-sql/library/versions.tmpl index 7c50b969d44..69abefd83e2 100644 --- a/libs/template/templates/dbt-sql/library/versions.tmpl +++ b/libs/template/templates/dbt-sql/library/versions.tmpl @@ -1,10 +1,5 @@ {{define "latest_lts_dbr_version" -}} - 15.4.x-scala2.12 -{{- end}} - -{{/* TODO: unused — no dbt-sql template references this macro. Remove it. */}} -{{define "latest_lts_db_connect_version_spec" -}} - >=16.4,<16.5 + 16.4.x-scala2.12 {{- end}} {{/* The serverless environment version to use.