From f01a6733ab5ac51e17015da581fc19e3cab97f0b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 6 May 2026 10:52:51 +0200 Subject: [PATCH 1/4] acceptance: add test for duplicate YAML merge key Co-authored-by: Isaac --- .../duplicate_yaml_merge_key/databricks.yml | 20 +++++++++++++++++++ .../duplicate_yaml_merge_key/out.test.toml | 3 +++ .../duplicate_yaml_merge_key/output.txt | 9 +++++++++ .../validate/duplicate_yaml_merge_key/script | 1 + 4 files changed, 33 insertions(+) create mode 100644 acceptance/bundle/validate/duplicate_yaml_merge_key/databricks.yml create mode 100644 acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml create mode 100644 acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt create mode 100644 acceptance/bundle/validate/duplicate_yaml_merge_key/script diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/databricks.yml b/acceptance/bundle/validate/duplicate_yaml_merge_key/databricks.yml new file mode 100644 index 00000000000..630455bc24f --- /dev/null +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +definitions: + cluster1: &cluster1 + num_workers: 1 + cluster2: &cluster2 + spark_version: "13.3.x-scala2.12" + +resources: + jobs: + my_job: + name: "test job" + tasks: + - task_key: "main" + new_cluster: + <<: *cluster1 + <<: *cluster2 + notebook_task: + notebook_path: "/notebook" diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt b/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt new file mode 100644 index 00000000000..7e68c5b1655 --- /dev/null +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate +Error: duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]' + in databricks.yml:18:13 + + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/script b/acceptance/bundle/validate/duplicate_yaml_merge_key/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/script @@ -0,0 +1 @@ +trace $CLI bundle validate From 8649f145c04e83880b982bf1611691e34c78d877 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 6 May 2026 10:52:58 +0200 Subject: [PATCH 2/4] fix: convert duplicate YAML merge key panic to diagnostic error Replacing the panic with a LocationError that carries the source location, then converting it to a diagnostic in LoadFromBytes so the output shows the file/line/column and a clear explanation of what is not allowed and how to fix it. Co-authored-by: Isaac --- bundle/config/root.go | 9 +++++++++ libs/dyn/yamlloader/loader.go | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 764d801bc27..6d4697cc1ba 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -3,6 +3,7 @@ package config import ( "bytes" "context" + "errors" "fmt" "os" "reflect" @@ -106,6 +107,14 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { // Load configuration tree from YAML. v, err := yamlloader.LoadYAML(path, bytes.NewBuffer(raw)) if err != nil { + var le *yamlloader.LocationError + if errors.As(err, &le) { + return nil, diag.Diagnostics{{ + Severity: diag.Error, + Summary: le.Summary, + Locations: []dyn.Location{le.Loc}, + }} + } return nil, diag.Errorf("failed to load %s: %v", path, err) } diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go index 79a4fb1d177..7ff4303d8ee 100644 --- a/libs/dyn/yamlloader/loader.go +++ b/libs/dyn/yamlloader/loader.go @@ -10,6 +10,17 @@ import ( "go.yaml.in/yaml/v3" ) +// LocationError is an error with a YAML source location that can be displayed +// to the user with a file path, line, and column number. +type LocationError struct { + Loc dyn.Location + Summary string +} + +func (e *LocationError) Error() string { + return fmt.Sprintf("yaml (%s): %s", e.Loc, e.Summary) +} + type loader struct { path string } @@ -110,7 +121,12 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro // However, when used as a key, it is treated as the string "null". case "!!merge": if merge != nil { - panic("merge node already set") + // The YAML merge key spec allows a single '<<' key per mapping. + // To merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'. + return dyn.InvalidValue, &LocationError{ + Loc: d.location(key), + Summary: "duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'", + } } merge = val continue From 6d3370456f9dd3325321ea3ccbe6c6922b00d111 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 6 May 2026 11:45:00 +0200 Subject: [PATCH 3/4] acceptance: verify suggested fix works in duplicate merge key test Co-authored-by: Isaac --- .../bundle/validate/duplicate_yaml_merge_key/output.txt | 9 ++++++++- .../bundle/validate/duplicate_yaml_merge_key/script | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt b/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt index 7e68c5b1655..655ed0cbe2a 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt @@ -6,4 +6,11 @@ Error: duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, u Found 1 error -Exit code: 1 +>>> [CLI] bundle validate +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/script b/acceptance/bundle/validate/duplicate_yaml_merge_key/script index 5350876150f..f0ce894ad29 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/script +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/script @@ -1 +1,6 @@ +musterr trace $CLI bundle validate + +update_file.py databricks.yml " <<: *cluster1 + <<: *cluster2" " <<: [*cluster1, *cluster2]" + trace $CLI bundle validate From 586bb3be16e8fa5fa54c40c72b72e1a3e9699ad1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 6 May 2026 13:02:31 +0200 Subject: [PATCH 4/4] acceptance: verify both anchors are merged in fixed YAML Co-authored-by: Isaac --- .../bundle/validate/duplicate_yaml_merge_key/output.txt | 4 ++++ acceptance/bundle/validate/duplicate_yaml_merge_key/script | 2 ++ 2 files changed, 6 insertions(+) diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt b/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt index 655ed0cbe2a..420ad818626 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/output.txt @@ -14,3 +14,7 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default Validation OK! +{ + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" +} diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/script b/acceptance/bundle/validate/duplicate_yaml_merge_key/script index f0ce894ad29..472434d5d5f 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/script +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/script @@ -4,3 +4,5 @@ update_file.py databricks.yml " <<: *cluster1 <<: *cluster2" " <<: [*cluster1, *cluster2]" trace $CLI bundle validate + +$CLI bundle validate -o json | jq '.resources.jobs.my_job.tasks[0].new_cluster'