Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NEXT_CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
### Notable Changes

### CLI
* Skip non-exportable objects (e.g., `MLFLOW_EXPERIMENT`) during `workspace export-dir` instead of failing ([#4081](https://github.com/databricks/cli/issues/4081))

### Bundles

Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@

>>> [CLI] workspace export-dir /test-dir [TEST_TMP_DIR]/export
Exporting files from /test-dir
Warning: /test-dir/file.py (skipped; file too large)

The following files were skipped because they exceed the maximum size limit:
- /test-dir/file.py (skipped; file too large)

/test-dir/file.py (skipped; file too large)
Export complete

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

>>> [CLI] workspace export-dir /test-dir [TEST_TMP_DIR]/export
Exporting files from /test-dir
/test-dir/experiment (skipped; cannot export MLFLOW_EXPERIMENT)
Export complete
2 changes: 2 additions & 0 deletions acceptance/cmd/workspace/export-dir-skip-experiments/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
mkdir -p "$TEST_TMP_DIR/export"
trace $CLI workspace export-dir /test-dir "$TEST_TMP_DIR/export"
29 changes: 29 additions & 0 deletions acceptance/cmd/workspace/export-dir-skip-experiments/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
Local = true
Cloud = false

[Env]
MSYS_NO_PATHCONV = "1"

[[Server]]
Pattern = "GET /api/2.0/workspace/list"
Response.Body = '''
{
"objects": [
{
"path": "/test-dir/experiment",
"object_type": "MLFLOW_EXPERIMENT",
"object_id": 125
}
]
}
'''

[[Server]]
Pattern = "GET /api/2.0/workspace/get-status"
Response.Body = '''
{
"path": "/test-dir",
"object_type": "DIRECTORY",
"object_id": 123
}
'''
44 changes: 29 additions & 15 deletions cmd/workspace/workspace/export_dir.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package workspace
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
Expand All @@ -24,7 +25,6 @@ type exportDirOptions struct {
sourceDir string
targetDir string
overwrite bool
warnings []string
}

// isFileSizeError checks if the error is due to file size limits.
Expand All @@ -51,6 +51,26 @@ func isFileSizeError(err error) bool {
return false
}

// Object types that cannot be exported via the workspace export API.
// These will be skipped with a warning during export-dir.
var nonExportableTypes = []workspace.ObjectType{
workspace.ObjectTypeLibrary,
workspace.ObjectTypeDashboard,
workspace.ObjectTypeRepo,
// MLFLOW_EXPERIMENT is not defined as a constant in the SDK
workspace.ObjectType("MLFLOW_EXPERIMENT"),
}

// isNonExportable checks if an object type cannot be exported.
func isNonExportable(objectType workspace.ObjectType) bool {
for _, t := range nonExportableTypes {
if objectType == t {
return true
}
}
return false
}

// The callback function exports the file specified at relPath. This function is
// meant to be used in conjunction with fs.WalkDir
func (opts *exportDirOptions) callback(ctx context.Context, workspaceFiler filer.Filer) func(string, fs.DirEntry, error) error {
Expand All @@ -77,6 +97,13 @@ func (opts *exportDirOptions) callback(ctx context.Context, workspaceFiler filer
return err
}
objectInfo := info.Sys().(workspace.ObjectInfo)

// Skip non-exportable objects (e.g., MLFLOW_EXPERIMENT, LIBRARY)
if isNonExportable(objectInfo.ObjectType) {
cmdio.LogString(ctx, fmt.Sprintf("%s (skipped; cannot export %s)", sourcePath, objectInfo.ObjectType))
return nil
}

targetPath += notebook.GetExtensionByLanguage(&objectInfo)

// Skip file if a file already exists in path.
Expand All @@ -92,9 +119,7 @@ func (opts *exportDirOptions) callback(ctx context.Context, workspaceFiler filer
if err != nil {
// Check if this is a file size limit error
if isFileSizeError(err) {
warning := sourcePath + " (skipped; file too large)"
cmdio.LogString(ctx, "Warning: "+warning)
opts.warnings = append(opts.warnings, warning)
cmdio.LogString(ctx, sourcePath+" (skipped; file too large)")
return nil
}
return err
Expand Down Expand Up @@ -140,7 +165,6 @@ func newExportDir() *cobra.Command {
w := cmdctx.WorkspaceClient(ctx)
opts.sourceDir = args[0]
opts.targetDir = args[1]
opts.warnings = []string{}

// Initialize a filer and a file system on the source directory
workspaceFiler, err := filer.NewWorkspaceFilesClient(w, opts.sourceDir)
Expand All @@ -159,16 +183,6 @@ func newExportDir() *cobra.Command {
return err
}

// Print all warnings at the end if any were collected
if len(opts.warnings) > 0 {
cmdio.LogString(ctx, "")
cmdio.LogString(ctx, "The following files were skipped because they exceed the maximum size limit:")
for _, warning := range opts.warnings {
cmdio.LogString(ctx, " - "+warning)
}
cmdio.LogString(ctx, "")
}

return cmdio.RenderWithTemplate(ctx, newExportCompletedEvent(opts.targetDir), "", "Export complete\n")
}

Expand Down