From 26401d2f84646723160249ed2fdbbbacd07e4839 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 6 May 2026 12:43:18 +0200 Subject: [PATCH 1/3] Replace old Spark Jar integration tests with an acceptance test --- .../deploy/spark-jar-task/databricks.yml.tmpl | 26 +++ .../myjar}/META-INF/MANIFEST.MF | 0 .../spark-jar-task/myjar}/PrintArgs.java | 0 .../deploy/spark-jar-task/out.test.toml | 3 + .../bundle/deploy/spark-jar-task/output.txt | 24 +++ .../bundle/deploy/spark-jar-task/script | 4 + .../bundle/deploy/spark-jar-task/test.toml | 17 ++ .../databricks_template_schema.json | 33 ---- .../template/databricks.yml.tmpl | 55 ------ integration/bundle/spark_jar_test.go | 156 ------------------ 10 files changed, 74 insertions(+), 244 deletions(-) create mode 100644 acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl rename {integration/bundle/bundles/spark_jar_task/template/{{.project_name}} => acceptance/bundle/deploy/spark-jar-task/myjar}/META-INF/MANIFEST.MF (100%) rename {integration/bundle/bundles/spark_jar_task/template/{{.project_name}} => acceptance/bundle/deploy/spark-jar-task/myjar}/PrintArgs.java (100%) create mode 100644 acceptance/bundle/deploy/spark-jar-task/out.test.toml create mode 100644 acceptance/bundle/deploy/spark-jar-task/output.txt create mode 100644 acceptance/bundle/deploy/spark-jar-task/script create mode 100644 acceptance/bundle/deploy/spark-jar-task/test.toml delete mode 100644 integration/bundle/bundles/spark_jar_task/databricks_template_schema.json delete mode 100644 integration/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl delete mode 100644 integration/bundle/spark_jar_test.go diff --git a/acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl b/acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl new file mode 100644 index 00000000000..c6e5c9ee324 --- /dev/null +++ b/acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: spark-jar-task-$UNIQUE_NAME + +artifacts: + my_java_code: + path: ./myjar + build: "javac PrintArgs.java && jar cvfm PrintArgs.jar META-INF/MANIFEST.MF PrintArgs.class" + files: + - source: ./myjar/PrintArgs.jar + +resources: + jobs: + jar_job: + name: "[${bundle.target}] Test Spark Jar Job $UNIQUE_NAME" + tasks: + - task_key: TestSparkJarTask + new_cluster: + num_workers: 1 + spark_version: 16.2.x-scala2.12 + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + data_security_mode: NONE + spark_jar_task: + main_class_name: PrintArgs + libraries: + - jar: ./myjar/PrintArgs.jar diff --git a/integration/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF b/acceptance/bundle/deploy/spark-jar-task/myjar/META-INF/MANIFEST.MF similarity index 100% rename from integration/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF rename to acceptance/bundle/deploy/spark-jar-task/myjar/META-INF/MANIFEST.MF diff --git a/integration/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java b/acceptance/bundle/deploy/spark-jar-task/myjar/PrintArgs.java similarity index 100% rename from integration/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java rename to acceptance/bundle/deploy/spark-jar-task/myjar/PrintArgs.java diff --git a/acceptance/bundle/deploy/spark-jar-task/out.test.toml b/acceptance/bundle/deploy/spark-jar-task/out.test.toml new file mode 100644 index 00000000000..bbc7fcfd1bd --- /dev/null +++ b/acceptance/bundle/deploy/spark-jar-task/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/spark-jar-task/output.txt b/acceptance/bundle/deploy/spark-jar-task/output.txt new file mode 100644 index 00000000000..3b556fbb60f --- /dev/null +++ b/acceptance/bundle/deploy/spark-jar-task/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle deploy +Building my_java_code... +Uploading myjar/PrintArgs.jar... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/spark-jar-task-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run jar_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Spark Jar Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Spark Jar Job [UNIQUE_NAME]" TERMINATED SUCCESS +Hello from Jar! +[] +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.jar_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/spark-jar-task-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/spark-jar-task/script b/acceptance/bundle/deploy/spark-jar-task/script new file mode 100644 index 00000000000..d736ca6bfae --- /dev/null +++ b/acceptance/bundle/deploy/spark-jar-task/script @@ -0,0 +1,4 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy +trace $CLI bundle run jar_job diff --git a/acceptance/bundle/deploy/spark-jar-task/test.toml b/acceptance/bundle/deploy/spark-jar-task/test.toml new file mode 100644 index 00000000000..86ea49e050c --- /dev/null +++ b/acceptance/bundle/deploy/spark-jar-task/test.toml @@ -0,0 +1,17 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + +Ignore = [ + 'myjar/PrintArgs.jar', + 'myjar/PrintArgs.class', +] + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{ + "run_id": 1234567890, + "logs": "Hello from Jar!\n[]" +} +''' diff --git a/integration/bundle/bundles/spark_jar_task/databricks_template_schema.json b/integration/bundle/bundles/spark_jar_task/databricks_template_schema.json deleted file mode 100644 index 1381da1ddc2..00000000000 --- a/integration/bundle/bundles/spark_jar_task/databricks_template_schema.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "properties": { - "project_name": { - "type": "string", - "default": "my_java_project", - "description": "Unique name for this project" - }, - "spark_version": { - "type": "string", - "description": "Spark version used for job cluster" - }, - "node_type_id": { - "type": "string", - "description": "Node type id for job cluster" - }, - "unique_id": { - "type": "string", - "description": "Unique ID for job name" - }, - "root": { - "type": "string", - "description": "Path to the root of the template" - }, - "artifact_path": { - "type": "string", - "description": "Path to the remote base path for artifacts" - }, - "instance_pool_id": { - "type": "string", - "description": "Instance pool id for job cluster" - } - } -} diff --git a/integration/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl b/integration/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl deleted file mode 100644 index db451cd93b1..00000000000 --- a/integration/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl +++ /dev/null @@ -1,55 +0,0 @@ -bundle: - name: spark-jar-task - -workspace: - root_path: "~/.bundle/{{.unique_id}}" - -artifacts: - my_java_code: - path: ./{{.project_name}} - build: "javac PrintArgs.java && jar cvfm PrintArgs.jar META-INF/MANIFEST.MF PrintArgs.class" - files: - - source: ./{{.project_name}}/PrintArgs.jar - -resources: - jobs: - jar_job: - name: "[${bundle.target}] Test Spark Jar Job {{.unique_id}}" - tasks: - - task_key: TestSparkJarTask - new_cluster: - num_workers: 1 - spark_version: "{{.spark_version}}" - node_type_id: "{{.node_type_id}}" - instance_pool_id: "{{.instance_pool_id}}" - spark_jar_task: - main_class_name: PrintArgs - libraries: - - jar: ./{{.project_name}}/PrintArgs.jar - -targets: - volume: - # Override the artifact path to upload artifacts to a volume path - workspace: - artifact_path: {{.artifact_path}} - - resources: - jobs: - jar_job: - tasks: - - task_key: TestSparkJarTask - new_cluster: - - # Force cluster to run in single user mode (force it to be a UC cluster) - data_security_mode: SINGLE_USER - - workspace: - resources: - jobs: - jar_job: - tasks: - - task_key: TestSparkJarTask - new_cluster: - - # Force cluster to run in no isolation mode (force it to be a non-UC cluster) - data_security_mode: NONE diff --git a/integration/bundle/spark_jar_test.go b/integration/bundle/spark_jar_test.go deleted file mode 100644 index 045af404fa5..00000000000 --- a/integration/bundle/spark_jar_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package bundle_test - -import ( - "context" - "testing" - - "github.com/databricks/cli/integration/internal/acc" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/env" - "github.com/google/uuid" - "github.com/stretchr/testify/require" -) - -// sparkJarTestCase defines a Databricks runtime version and a local Java version requirement -type sparkJarTestCase struct { - name string // Test name - runtimeVersion string // The Spark runtime version to test - requiredJavaVersion string // Java version that can compile jar to pass this test -} - -// runSparkJarTests runs a set of test cases with appropriate Java version checks -// testRunner is the function that runs the actual test with the runtime version -func runSparkJarTests(t *testing.T, testCases []sparkJarTestCase, testRunner func(t *testing.T, runtimeVersion string)) { - t.Helper() - - testCanRun := make(map[string]bool) - atLeastOneCanRun := false - for _, tc := range testCases { - if testutil.HasJDK(t, t.Context(), tc.requiredJavaVersion) { - testCanRun[tc.name] = true - atLeastOneCanRun = true - continue - } - testCanRun[tc.name] = false - } - - if !atLeastOneCanRun { - t.Fatal("At least one test is required to pass. All tests were skipped because no compatible Java version was found.") - } - - // Run the tests that can run - for _, tc := range testCases { - canRun := testCanRun[tc.name] - - t.Run(tc.name, func(t *testing.T) { - if !canRun { - t.Skipf("Skipping %s: requires Java version %v", tc.name, tc.requiredJavaVersion) - return - } - - t.Parallel() - testRunner(t, tc.runtimeVersion) - }) - } -} - -func runSparkJarTestCommon(t *testing.T, ctx context.Context, sparkVersion, artifactPath string) { - nodeTypeId := testutil.GetCloud(t).NodeTypeID() - tmpDir := t.TempDir() - instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") - bundleRoot := initTestTemplateWithBundleRoot(t, ctx, "spark_jar_task", map[string]any{ - "node_type_id": nodeTypeId, - "unique_id": uuid.New().String(), - "spark_version": sparkVersion, - "root": tmpDir, - "artifact_path": artifactPath, - "instance_pool_id": instancePoolId, - }, tmpDir) - - deployBundle(t, ctx, bundleRoot) - - t.Cleanup(func() { - destroyBundle(t, context.WithoutCancel(ctx), bundleRoot) - }) - - if testing.Short() { - t.Log("Skip the job run in short mode") - return - } - - out, err := runResource(t, ctx, bundleRoot, "jar_job") - require.NoError(t, err) - require.Contains(t, out, "Hello from Jar!") -} - -func runSparkJarTestFromVolume(t *testing.T, sparkVersion string) { - ctx, wt := acc.UcWorkspaceTest(t) - volumePath := acc.TemporaryVolume(wt) - ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "volume") - runSparkJarTestCommon(t, ctx, sparkVersion, volumePath) -} - -func runSparkJarTestFromWorkspace(t *testing.T, sparkVersion string) { - ctx, _ := acc.WorkspaceTest(t) - ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "workspace") - runSparkJarTestCommon(t, ctx, sparkVersion, "n/a") -} - -func TestSparkJarTaskDeployAndRunOnVolumes(t *testing.T) { - // Failure on earlier DBR versions: - // - // JAR installation from Volumes is supported on UC Clusters with DBR >= 13.3. - // Denied library is Jar(/Volumes/main/test-schema-ldgaklhcahlg/my-volume/.internal/PrintArgs.jar) - // - - testCases := []sparkJarTestCase{ - { - name: "Databricks Runtime 13.3 LTS", - runtimeVersion: "13.3.x-scala2.12", // 13.3 LTS (includes Apache Spark 3.4.1, Scala 2.12) - requiredJavaVersion: "1.8.0", // Only JDK 8 is supported - }, - { - name: "Databricks Runtime 14.3 LTS", - runtimeVersion: "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) - requiredJavaVersion: "1.8.0", // Only JDK 8 is supported - }, - { - name: "Databricks Runtime 15.4 LTS", - runtimeVersion: "15.4.x-scala2.12", // 15.4 LTS (includes Apache Spark 3.5.0, Scala 2.12) - requiredJavaVersion: "1.8.0", // Only JDK 8 is supported - }, - { - name: "Databricks Runtime 16.2", - runtimeVersion: "16.2.x-scala2.12", // 16.2 (includes Apache Spark 3.5.2, Scala 2.12) - requiredJavaVersion: "11.0", // Can run jars compiled by Java 11 - }, - } - runSparkJarTests(t, testCases, runSparkJarTestFromVolume) -} - -func TestSparkJarTaskDeployAndRunOnWorkspace(t *testing.T) { - // Failure on earlier DBR versions: - // - // Library from /Workspace is not allowed on this cluster. - // Please switch to using DBR 14.1+ No Isolation Shared or DBR 13.1+ Shared cluster or 13.2+ Assigned cluster to use /Workspace libraries. - // - - testCases := []sparkJarTestCase{ - { - name: "Databricks Runtime 14.3 LTS", - runtimeVersion: "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) - requiredJavaVersion: "1.8.0", // Only JDK 8 is supported - }, - { - name: "Databricks Runtime 15.4 LTS", - runtimeVersion: "15.4.x-scala2.12", // 15.4 LTS (includes Apache Spark 3.5.0, Scala 2.12) - requiredJavaVersion: "1.8.0", // Only JDK 8 is supported - }, - { - name: "Databricks Runtime 16.2", - runtimeVersion: "16.2.x-scala2.12", // 16.2 (includes Apache Spark 3.5.2, Scala 2.12) - requiredJavaVersion: "11.0", // Can run jars compiled by Java 11 - }, - } - runSparkJarTests(t, testCases, runSparkJarTestFromWorkspace) -} From 642bfff85089ee2bd01c89ca68570b73b62b1ac2 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 6 May 2026 13:33:50 +0200 Subject: [PATCH 2/3] fix lint --- integration/bundle/helpers_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/integration/bundle/helpers_test.go b/integration/bundle/helpers_test.go index 48ca44585b2..712038d0e0f 100644 --- a/integration/bundle/helpers_test.go +++ b/integration/bundle/helpers_test.go @@ -67,13 +67,6 @@ func deployBundle(t testutil.TestingT, ctx context.Context, path string) { require.NoError(t, err) } -func runResource(t testutil.TestingT, ctx context.Context, path, key string) (string, error) { - ctx = env.Set(ctx, "BUNDLE_ROOT", path) - c := testcli.NewRunner(t, ctx, "bundle", "run", key) - stdout, _, err := c.Run() - return stdout.String(), err -} - func destroyBundle(t testutil.TestingT, ctx context.Context, path string) { ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := testcli.NewRunner(t, ctx, "bundle", "destroy", "--auto-approve") From 073b42213364d9b42a769c534697002aaf72235a Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 6 May 2026 13:43:52 +0200 Subject: [PATCH 3/3] cleanup + lts --- .../deploy/spark-jar-task/databricks.yml.tmpl | 2 +- integration/internal/acc/workspace.go | 29 ------------ internal/testutil/jdk.go | 44 ------------------- 3 files changed, 1 insertion(+), 74 deletions(-) delete mode 100644 internal/testutil/jdk.go diff --git a/acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl b/acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl index c6e5c9ee324..9078bcbdbbd 100644 --- a/acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl +++ b/acceptance/bundle/deploy/spark-jar-task/databricks.yml.tmpl @@ -16,7 +16,7 @@ resources: - task_key: TestSparkJarTask new_cluster: num_workers: 1 - spark_version: 16.2.x-scala2.12 + spark_version: 16.4.x-scala2.12 node_type_id: $NODE_TYPE_ID instance_pool_id: $TEST_INSTANCE_POOL_ID data_security_mode: NONE diff --git a/integration/internal/acc/workspace.go b/integration/internal/acc/workspace.go index 878d4526568..60804509124 100644 --- a/integration/internal/acc/workspace.go +++ b/integration/internal/acc/workspace.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/require" @@ -40,34 +39,6 @@ func WorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { return wt.ctx, wt } -// Run the workspace test only on UC workspaces. -func UcWorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { - t.Helper() - testutil.LoadDebugEnvIfRunFromIDE(t, "workspace") - - t.Logf("CLOUD_ENV=%s", testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) - - if env.Get(t.Context(), "TEST_METASTORE_ID") == "" { - t.Skipf("Skipping on non-UC workspaces") - } - if env.Get(t.Context(), "DATABRICKS_ACCOUNT_ID") != "" { - t.Skipf("Skipping on accounts") - } - - w, err := databricks.NewWorkspaceClient() - require.NoError(t, err) - - wt := &WorkspaceT{ - TestingT: t, - - W: w, - - ctx: t.Context(), - } - - return wt.ctx, wt -} - func (t *WorkspaceT) TestClusterID() string { t.Helper() clusterID := testutil.GetEnvOrSkipTest(t, "TEST_BRICKS_CLUSTER_ID") diff --git a/internal/testutil/jdk.go b/internal/testutil/jdk.go deleted file mode 100644 index 6c32777190e..00000000000 --- a/internal/testutil/jdk.go +++ /dev/null @@ -1,44 +0,0 @@ -package testutil - -import ( - "context" - "os/exec" - "strings" -) - -// HasJDK checks if the specified Java version is available in the system. -// It returns true if the required JDK version is present, false otherwise. -// This is a non-failing variant of RequireJDK. -// -// Example output of `java -version` in eclipse-temurin:8: -// openjdk version "1.8.0_442" -// OpenJDK Runtime Environment (Temurin)(build 1.8.0_442-b06) -// OpenJDK 64-Bit Server VM (Temurin)(build 25.442-b06, mixed mode) -// -// Example output of `java -version` in java11 (homebrew): -// openjdk version "11.0.26" 2025-01-21 -// OpenJDK Runtime Environment Homebrew (build 11.0.26+0) -// OpenJDK 64-Bit Server VM Homebrew (build 11.0.26+0, mixed mode) -func HasJDK(t TestingT, ctx context.Context, version string) bool { - t.Helper() - - // Try to execute "java -version" command - cmd := exec.CommandContext(ctx, "java", "-version") - output, err := cmd.CombinedOutput() - if err != nil { - t.Logf("Failed to execute java -version: %v", err) - return false - } - - javaVersionOutput := string(output) - - // Check if the output contains the expected version - expectedVersionString := "version \"" + version - if strings.Contains(javaVersionOutput, expectedVersionString) { - t.Logf("Detected JDK version %s", version) - return true - } - - t.Logf("Required JDK version %s not found, instead got: %s", version, javaVersionOutput) - return false -}