Replace regex-based interpolation with character scanner

NEXT_CHANGELOG.md

@@ -8,4 +8,6 @@
 
 * Fixed `--force-pull` on `bundle summary` and `bundle open` so the flag bypasses the local state cache and reads state from the workspace.
 
+* Replace regex-based variable interpolation with a character scanner ([#4747](https://github.com/databricks/cli/pull/4747)).
+
 ### Dependency updates

acceptance/bundle/resource_deps/bad_syntax/out.deploy.direct.txt

@@ -1,3 +1,7 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Deploying resources...
 Updating deployment state...

acceptance/bundle/resource_deps/bad_syntax/out.plan.direct.txt

@@ -1,3 +1,7 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 create volumes.bar
 create volumes.foo
 

acceptance/bundle/resource_deps/bad_syntax/out.plan.terraform.txt

@@ -1,3 +1,7 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 Error: exit status 1
 
 Error: Invalid attribute name

acceptance/bundle/resource_deps/bad_syntax/output.txt

@@ -1,5 +1,9 @@
 
 >>> [CLI] bundle validate -o json
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 {
   "volumes": {
     "bar": {

acceptance/bundle/variables/dollar_before_non_brace/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '$foo is not a reference'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/dollar_before_non_brace/output.txt

@@ -0,0 +1,9 @@
+
+>>> [CLI] bundle validate
+Name: $foo is not a reference
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/$foo is not a reference/default
+
+Validation OK!

acceptance/bundle/variables/dollar_before_non_brace/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/empty_ref/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '${}'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/empty_ref/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: empty variable reference
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${}
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${}/default
+
+Found 1 warning

acceptance/bundle/variables/empty_ref/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/leading_digit_ref/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '${0foo}'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/leading_digit_ref/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: invalid variable reference ${0foo}: invalid key "0foo"
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${0foo}
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${0foo}/default
+
+Found 1 warning

acceptance/bundle/variables/leading_digit_ref/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/malformed_reference/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: "${foo.bar-}"
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/malformed_reference/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: invalid variable reference ${foo.bar-}: invalid key "bar-"
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${foo.bar-}
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${foo.bar-}/default
+
+Found 1 warning

acceptance/bundle/variables/malformed_reference/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/unterminated_ref/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '${foo.bar'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/unterminated_ref/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: unterminated variable reference
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${foo.bar
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${foo.bar/default
+
+Found 1 warning

acceptance/bundle/variables/unterminated_ref/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/var_in_var_3level/databricks.yml

@@ -0,0 +1,12 @@
+bundle:
+  name: test-bundle
+
+variables:
+  env:
+    default: prod
+  region_prod:
+    default: us
+  endpoint_us:
+    default: https://us.example.com
+  result:
+    default: ${var.endpoint_${var.region_${var.env}}}

acceptance/bundle/variables/var_in_var_3level/output.txt

@@ -0,0 +1,20 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "endpoint_us": {
+    "default": "https://us.example.com",
+    "value": "https://us.example.com"
+  },
+  "env": {
+    "default": "prod",
+    "value": "prod"
+  },
+  "region_prod": {
+    "default": "us",
+    "value": "us"
+  },
+  "result": {
+    "default": "https://us.example.com",
+    "value": "https://us.example.com"
+  }
+}

acceptance/bundle/variables/var_in_var_3level/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq .variables

bundle/config/mutator/warn_malformed_references.go

@@ -0,0 +1,64 @@
+package mutator
+
+import (
+	"context"
+	"errors"
+	"slices"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/interpolation"
+)
+
+type warnMalformedReferences struct{}
+
+// WarnMalformedReferences returns a mutator that emits warnings for strings
+// containing malformed variable references (e.g. "${foo.bar-}").
+func WarnMalformedReferences() bundle.Mutator {
+	return &warnMalformedReferences{}
+}
+
+func (*warnMalformedReferences) Name() string {
+	return "WarnMalformedReferences"
+}
+
+func (*warnMalformedReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	_, err := dyn.Walk(b.Config.Value(), func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+		// Only check values with source locations to avoid false positives
+		// from synthesized/computed values.
+		if len(v.Locations()) == 0 {
+			return v, nil
+		}
+
+		s, ok := v.AsString()
+		if !ok {
+			return v, nil
+		}
+
+		var pe *interpolation.ParseError
+		if _, parseErr := interpolation.Parse(s); !errors.As(parseErr, &pe) {
+			return v, nil
+		}
+
+		// Clone locations and adjust column with the position offset
+		// so the diagnostic points to the problematic reference.
+		locs := slices.Clone(v.Locations())
+		if len(locs) > 0 {
+			locs[0].Column += pe.Pos
+		}
+
+		diags = append(diags, diag.Diagnostic{
+			Severity:  diag.Warning,
+			Summary:   pe.Msg,
+			Locations: locs,
+			Paths:     []dyn.Path{p},
+		})
+		return v, nil
+	})
+	if err != nil {
+		diags = diags.Extend(diag.FromErr(err))
+	}
+	return diags
+}

bundle/configsync/variables.go

@@ -11,6 +11,7 @@ import (
 	"github.com/databricks/cli/bundle/direct/dstate"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/dynvar"
+	"github.com/databricks/cli/libs/interpolation"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/structs/structpath"
 )
@@ -500,50 +501,36 @@ func parseTemplateSegments(template string, resolved dyn.Value) []templateSegmen
 	}
 
 	var segments []templateSegment
-	cursor := 0
 
-	for _, m := range ref.Matches {
-		fullMatch := m[0]
-
-		idx := strings.Index(template[cursor:], fullMatch)
-		if idx < 0 {
-			return nil
-		}
-
-		if idx > 0 {
+	for _, tok := range ref.Tokens {
+		switch tok.Kind {
+		case interpolation.TokenLiteral:
 			segments = append(segments, templateSegment{
-				raw: template[cursor : cursor+idx],
+				raw: tok.Value,
 			})
-		}
+		case interpolation.TokenRef:
+			fullMatch := "${" + tok.Value + "}"
+			resolvedPath, ok := resolveReferencePath(fullMatch)
+			if !ok {
+				return nil
+			}
 
-		resolvedPath, ok := resolveReferencePath(fullMatch)
-		if !ok {
-			return nil
-		}
+			resolvedV, err := dyn.GetByPath(resolved, resolvedPath)
+			if err != nil {
+				return nil
+			}
 
-		resolvedV, err := dyn.GetByPath(resolved, resolvedPath)
-		if err != nil {
-			return nil
-		}
+			resolvedStr, ok := resolvedV.AsString()
+			if !ok {
+				return nil
+			}
 
-		resolvedStr, ok := resolvedV.AsString()
-		if !ok {
-			return nil
+			segments = append(segments, templateSegment{
+				raw:           fullMatch,
+				isVariable:    true,
+				resolvedValue: resolvedStr,
+			})
 		}
-
-		segments = append(segments, templateSegment{
-			raw:           fullMatch,
-			isVariable:    true,
-			resolvedValue: resolvedStr,
-		})
-
-		cursor += idx + len(fullMatch)
-	}
-
-	if cursor < len(template) {
-		segments = append(segments, templateSegment{
-			raw: template[cursor:],
-		})
 	}
 
 	return segments

bundle/phases/initialize.go

@@ -95,6 +95,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) {
 		// searches for strings with variable references in them.
 		mutator.RewriteWorkspacePrefix(),
 
+		// Walks the config tree and emits warnings for malformed variable references
+		// (e.g. "${foo.bar-}") before variable resolution occurs.
+		mutator.WarnMalformedReferences(),
+
 		// Collect telemetry on $${} and \${} escape patterns before variable resolution.
 		mutator.CollectEscapeTelemetry(),