Skip large files during export-dir instead of failing (#3984)

Fixes #3691 

Skip large files in workspace export-dir with warnings instead of
failing

## Changes

- Modified `workspace export-dir` to handle files exceeding the 10MB
size limit gracefully
- When a file exceeds the limit, log a warning and continue exporting
other files
- Display a summary of all skipped files at the end of the export
- Added integration test `TestExportDirSkipsLargeFiles` to verify the
behavior

## Why

Previously, when exporting a directory containing files larger than 10MB
(the workspace API limit), the entire command would fail with an error.
This made it impossible to export directories containing a mix of small
and large files. Users would have to manually identify and exclude large
files before running the export.

Now the command skips oversized files with a clear warning, allowing
users to export what they can and see which files were skipped in a
summary at the end.

## Tests

- Added integration test `TestExportDirSkipsLargeFiles` that:
  - Creates an 11MB file (exceeds 10MB limit) and smaller files
  - Verifies large file is skipped with appropriate warnings in stderr
  - Verifies smaller files are exported successfully
  - Verifies no empty file is created locally for skipped files

Author: varundeepsaini

acceptance/cmd/workspace/export-dir-file-size-limit/out.test.toml

@@ -0,0 +1,9 @@
+
+>>> [CLI] workspace export-dir /test-dir [TEST_TMP_DIR]/export
+Exporting files from /test-dir
+Warning: /test-dir/file.py (skipped; file too large)
+
+The following files were skipped because they exceed the maximum size limit:
+  - /test-dir/file.py (skipped; file too large)
+
+Export complete

acceptance/cmd/workspace/export-dir-file-size-limit/output.txt

@@ -0,0 +1,4 @@
+$CLI workspace import /test-dir/file.py --file test_file.py --format AUTO --language PYTHON
+
+mkdir -p "$TEST_TMP_DIR/export"
+trace $CLI workspace export-dir /test-dir "$TEST_TMP_DIR/export"

acceptance/cmd/workspace/export-dir-file-size-limit/script

@@ -0,0 +1,33 @@
+Local = true
+Cloud = false
+
+[Env]
+MSYS_NO_PATHCONV = "1"
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/list"
+Response.Body = '''
+{
+  "objects": [
+    {
+      "path": "/test-dir/file.py",
+      "object_type": "NOTEBOOK",
+      "language": "PYTHON",
+      "object_id": 123,
+      "size": 1000,
+      "created_at": 1609459200000,
+      "modified_at": 1609459200000
+    }
+  ]
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/export"
+Response.StatusCode = 400
+Response.Body = '''
+{
+  "error_code": "MAX_NOTEBOOK_SIZE_EXCEEDED",
+  "message": "Notebook size exceeds maximum allowed size"
+}
+'''

acceptance/cmd/workspace/export-dir-file-size-limit/test.toml

@@ -0,0 +1 @@
+print("hello")

acceptance/cmd/workspace/export-dir-file-size-limit/test_file.py

@@ -2,8 +2,10 @@ package workspace
 
 import (
 	"context"
+	"errors"
 	"io"
 	"io/fs"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -13,6 +15,7 @@ import (
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/notebook"
+	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 	"github.com/spf13/cobra"
 )
@@ -21,11 +24,36 @@ type exportDirOptions struct {
 	sourceDir string
 	targetDir string
 	overwrite bool
+	warnings  []string
+}
+
+// isFileSizeError checks if the error is due to file size limits.
+func isFileSizeError(err error) bool {
+	var aerr *apierr.APIError
+	if !errors.As(err, &aerr) || aerr.StatusCode != http.StatusBadRequest {
+		return false
+	}
+
+	// Check ErrorCode field
+	if aerr.ErrorCode == "MAX_NOTEBOOK_SIZE_EXCEEDED" || aerr.ErrorCode == "MAX_READ_SIZE_EXCEEDED" {
+		return true
+	}
+
+	// Check ErrorInfo.Reason field
+	details := aerr.ErrorDetails()
+	if details.ErrorInfo != nil {
+		reason := details.ErrorInfo.Reason
+		if reason == "MAX_NOTEBOOK_SIZE_EXCEEDED" || reason == "MAX_READ_SIZE_EXCEEDED" {
+			return true
+		}
+	}
+
+	return false
 }
 
 // The callback function exports the file specified at relPath. This function is
 // meant to be used in conjunction with fs.WalkDir
-func (opts exportDirOptions) callback(ctx context.Context, workspaceFiler filer.Filer) func(string, fs.DirEntry, error) error {
+func (opts *exportDirOptions) callback(ctx context.Context, workspaceFiler filer.Filer) func(string, fs.DirEntry, error) error {
 	sourceDir := opts.sourceDir
 	targetDir := opts.targetDir
 	overwrite := opts.overwrite
@@ -59,18 +87,27 @@ func (opts exportDirOptions) callback(ctx context.Context, workspaceFiler filer.
 			return cmdio.RenderWithTemplate(ctx, newFileSkippedEvent(relPath, targetPath), "", "{{.SourcePath}} -> {{.TargetPath}} (skipped; already exists)\n")
 		}
 
-		// create the file
-		f, err := os.Create(targetPath)
+		// Write content to the local file
+		r, err := workspaceFiler.Read(ctx, relPath)
 		if err != nil {
+			// Check if this is a file size limit error
+			if isFileSizeError(err) {
+				warning := sourcePath + " (skipped; file too large)"
+				cmdio.LogString(ctx, "Warning: "+warning)
+				opts.warnings = append(opts.warnings, warning)
+				return nil
+			}
 			return err
 		}
-		defer f.Close()
+		defer r.Close()
 
-		// Write content to the local file
-		r, err := workspaceFiler.Read(ctx, relPath)
+		// create the file
+		f, err := os.Create(targetPath)
 		if err != nil {
 			return err
 		}
+		defer f.Close()
+
 		_, err = io.Copy(f, r)
 		if err != nil {
 			return err
@@ -103,6 +140,7 @@ func newExportDir() *cobra.Command {
 		w := cmdctx.WorkspaceClient(ctx)
 		opts.sourceDir = args[0]
 		opts.targetDir = args[1]
+		opts.warnings = []string{}
 
 		// Initialize a filer and a file system on the source directory
 		workspaceFiler, err := filer.NewWorkspaceFilesClient(w, opts.sourceDir)
@@ -120,6 +158,17 @@ func newExportDir() *cobra.Command {
 		if err != nil {
 			return err
 		}
+
+		// Print all warnings at the end if any were collected
+		if len(opts.warnings) > 0 {
+			cmdio.LogString(ctx, "")
+			cmdio.LogString(ctx, "The following files were skipped because they exceed the maximum size limit:")
+			for _, warning := range opts.warnings {
+				cmdio.LogString(ctx, "  - "+warning)
+			}
+			cmdio.LogString(ctx, "")
+		}
+
 		return cmdio.RenderWithTemplate(ctx, newExportCompletedEvent(opts.targetDir), "", "Export complete\n")
 	}