Move dev-tool lock dirs from source into into build folder

Signed-off-by: Marek Kubica <[email protected]>

Author: Leonidas-from-XIV

src/dune_pkg/lock_dir.ml

@@ -1789,9 +1789,18 @@ let loc_in_source_tree loc =
   loc
   |> Loc.map_pos ~f:(fun ({ pos_fname; _ } as pos) ->
     let path = Path.of_string pos_fname in
-    let new_path = in_source_tree path in
-    let pos_fname = Path.Source.to_string new_path in
-    { pos with pos_fname })
+    match Path.of_string pos_fname with
+    | External _ | In_source_tree _ -> pos
+    | In_build_dir b ->
+      (match Path.Build.explode b with
+       | ".dev-tools.locks" :: _ ->
+         (* we're excluding the hidden dev-tools.locks folders in the build folder
+          from rewriting  *)
+         pos
+       | _otherwise ->
+         let new_path = in_source_tree path in
+         let pos_fname = Path.Source.to_string new_path in
+         { pos with pos_fname }))
 ;;
 
 let check_if_solved_for_platform { solved_for_platforms; _ } ~platform =

src/dune_rules/format_rules.ml

@@ -33,8 +33,7 @@ end
 
 module Ocamlformat = struct
   let dev_tool_lock_dir_exists () =
-    let path = Lock_dir.dev_tool_source_lock_dir Ocamlformat in
-    Source_tree.find_dir path >>| Option.is_some
+    Lock_dir.dev_tool_source_lock_dir Ocamlformat |> Path.source |> Path.Untracked.exists
   ;;
 
   (* Config files for ocamlformat. When these are changed, running
@@ -129,7 +128,7 @@ let gen_rules_output
   let loc = Format_config.loc config in
   let dir = Path.Build.parent_exn output_dir in
   let alias_formatted = Alias.fmt ~dir:output_dir in
-  let* ocamlformat_is_locked = Ocamlformat.dev_tool_lock_dir_exists () in
+  let ocamlformat_is_locked = Ocamlformat.dev_tool_lock_dir_exists () in
   let setup_formatting file =
     (let input_basename = Path.Source.basename file in
      let input = Path.Build.relative dir input_basename in

src/dune_rules/lock_dir.ml

@@ -148,7 +148,9 @@ let dev_tool_to_path_segment dev_tool =
 ;;
 
 let dev_tool_source_lock_dir dev_tool =
-  let dev_tools_path = Path.Source.(relative root "dev-tools.locks") in
+  let dev_tools_path =
+    Path.Source.L.relative Path.Source.root [ "_build"; ".dev-tools.locks" ]
+  in
   let dev_tool_segment = dev_tool_to_path_segment dev_tool in
   Path.Source.append_local dev_tools_path dev_tool_segment
 ;;

src/dune_rules/lock_rules.ml

@@ -451,12 +451,60 @@ let copy_lock_dir ~target ~lock_dir ~deps ~files =
             ~dirs:(Path.Build.Set.singleton target))
 ;;
 
-let setup_copy_rules ~dir:target ~lock_dir =
-  let+ deps, files = Source_deps.files (Path.source lock_dir) in
+let files dir =
+  let rec recurse dir =
+    match Path.Untracked.readdir_unsorted_with_kinds dir with
+    | Ok entries ->
+      entries
+      |> List.fold_left
+           ~init:(Path.Set.empty, Path.Set.empty)
+           ~f:(fun (files, empty_directories) (entry, kind) ->
+             let path = Path.relative dir entry in
+             match (kind : Unix.file_kind) with
+             | S_REG ->
+               let files = Path.Set.add files path in
+               files, empty_directories
+             | S_DIR ->
+               let files', empty_directories' = recurse path in
+               (match Path.Set.is_empty files', Path.Set.is_empty empty_directories' with
+                | true, true ->
+                  let empty_directories = Path.Set.add empty_directories path in
+                  files, empty_directories
+                | _, _ ->
+                  let files = Path.Set.union files files' in
+                  let empty_directories =
+                    Path.Set.union empty_directories empty_directories'
+                  in
+                  files, empty_directories)
+             | otherwise ->
+               Code_error.raise
+                 "unsupported kind of file in folder"
+                 [ "path", Path.to_dyn path; "kind", File_kind.to_dyn otherwise ])
+    | Error (ENOENT, _, _) -> Path.Set.empty, Path.Set.empty
+    | Error unix_error ->
+      User_error.raise
+        [ Pp.textf
+            "Failed to read lock dir files of %s:"
+            (Path.to_string_maybe_quoted dir)
+        ; Pp.text (Unix_error.Detailed.to_string_hum unix_error)
+        ]
+  in
+  let files, empty_directories = recurse dir in
+  Dep.Set.of_source_files ~files ~empty_directories, files
+;;
+
+let setup_copy_rules ~dir:target ~assume_src_exists ~lock_dir =
+  let+ () = Memo.return () in
+  let deps, files = files (Path.source lock_dir) in
   let directory_targets, rules =
     match Path.Set.is_empty files with
     | true -> Path.Build.Map.empty, Rules.empty
     | false ->
+      let deps =
+        match assume_src_exists with
+        | false -> deps
+        | true -> Dep.Set.empty
+      in
       let directory_targets = Path.Build.Map.singleton target Loc.none in
       let { Action_builder.With_targets.build; targets } =
         copy_lock_dir ~target ~lock_dir ~deps ~files
@@ -478,7 +526,7 @@ let setup_lock_rules_with_source (workspace : Workspace.t) ~dir ~lock_dir =
   match source with
   | `Source_tree lock_dir ->
     let dir = Path.Build.append_source dir lock_dir in
-    setup_copy_rules ~dir ~lock_dir
+    setup_copy_rules ~assume_src_exists:false ~dir ~lock_dir
   | `Generated -> Memo.return (setup_lock_rules ~dir ~lock_dir)
 ;;
 
@@ -487,7 +535,9 @@ let setup_dev_tool_lock_rules ~dir dev_tool =
   let dev_tool_name = Dune_lang.Package_name.to_string package_name in
   let dir = Path.Build.relative dir dev_tool_name in
   let lock_dir = Lock_dir.dev_tool_source_lock_dir dev_tool in
-  setup_copy_rules ~dir ~lock_dir
+  (* dev tool lock files are created in _build outside of the build system
+     so we have to tell the build system not to try to create them *)
+  setup_copy_rules ~dir ~assume_src_exists:true ~lock_dir
 ;;
 
 let setup_rules ~components ~dir =

src/dune_rules/pkg_rules.ml

@@ -1494,10 +1494,10 @@ end = struct
             in
             resolve db dep_loc dep_pkg_digest package_universe)
       and+ files_dir =
-        let* lock_dir =
+        let+ lock_dir =
           Package_universe.lock_dir_path package_universe >>| Option.value_exn
         in
-        let+ files_dir =
+        let files_dir =
           let module Pkg = Dune_pkg.Lock_dir.Pkg in
           (* TODO(steve): simplify this once portable lockdirs become the
              default. This logic currently handles both the cases where
@@ -1509,17 +1509,15 @@ end = struct
           let path_with_version =
             Pkg.source_files_dir info.name (Some info.version) ~lock_dir
           in
-          let* path_with_version_exists =
-            Fs_memo.dir_exists (Path.Outside_build_dir.In_source_dir path_with_version)
+          let path_with_version_exists =
+            Path.Untracked.exists (Path.source path_with_version)
           in
           match path_with_version_exists with
-          | true ->
-            Memo.return @@ Some (Pkg.files_dir info.name (Some info.version) ~lock_dir)
+          | true -> Some (Pkg.files_dir info.name (Some info.version) ~lock_dir)
           | false ->
             let path_without_version = Pkg.source_files_dir info.name None ~lock_dir in
-            let+ path_without_version_exists =
-              Fs_memo.dir_exists
-                (Path.Outside_build_dir.In_source_dir path_without_version)
+            let path_without_version_exists =
+              Path.Untracked.exists (Path.source path_without_version)
             in
             (match path_without_version_exists with
              | true -> Some (Pkg.files_dir info.name None ~lock_dir)
@@ -1533,18 +1531,14 @@ end = struct
               "Package files directory is external source directory, this is unsupported"
               [ "dir", Path.External.to_dyn e ]
           | In_source_tree s ->
-            (match Path.Source.explode s with
-             | [ "dev-tools.locks"; dev_tool; files_dir ] ->
+            Code_error.raise "Unexpected files_dir path" [ "dir", Path.Source.to_dyn s ]
+          | In_build_dir b ->
+            (match Path.Build.explode b with
+             | [ ".dev-tools.locks"; dev_tool; files_dir ] ->
                Path.Build.L.relative
                  Private_context.t.build_dir
                  [ "default"; ".dev-tool-locks"; dev_tool; files_dir ]
-             | otherwise ->
-               Code_error.raise
-                 "Unexpected files_dir path"
-                 [ "components", (Dyn.list Dyn.string) otherwise ])
-          | In_build_dir b ->
-            (* it's already a build path, no need to do anything *)
-            b)
+             | _otherwise -> b))
       in
       let id = Pkg.Id.gen () in
       let write_paths =

test/blackbox-tests/test-cases/pkg/ocamlformat/gh10991.t

@@ -19,7 +19,7 @@ Initial file:
   let () = print_endline "Hello, world"
 
   $ DUNE_CONFIG__LOCK_DEV_TOOL=enabled dune fmt
-  Solution for dev-tools.locks/ocamlformat:
+  Solution for _build/.dev-tools.locks/ocamlformat:
   - ocamlformat.0.0.1
   File "foo.ml", line 1, characters 0-0:
   Error: Files _build/default/foo.ml and _build/default/.formatted/foo.ml

test/blackbox-tests/test-cases/pkg/ocamlformat/gh11037.t

@@ -40,7 +40,7 @@ attempt to build the package "foo".
   $ cat foo.ml
   let () = print_endline "Hello, world"
   $ DUNE_CONFIG__LOCK_DEV_TOOL=enabled dune fmt
-  Solution for dev-tools.locks/ocamlformat:
+  Solution for _build/.dev-tools.locks/ocamlformat:
   - ocamlformat.0.0.1
   File "foo.ml", line 1, characters 0-0:
   Error: Files _build/default/foo.ml and _build/default/.formatted/foo.ml

test/blackbox-tests/test-cases/pkg/ocamlformat/helpers.sh

@@ -1,5 +1,7 @@
 . ../helpers.sh
 
+dev_tool_lock_dir="_build/.dev-tools.locks/ocamlformat"
+
 make_fake_ocamlformat() {
   version=$1
   if [ "$#" -eq "1" ]
@@ -97,7 +99,7 @@ EOF
 (lang dune 3.20)
 
 (lock_dir
- (path "dev-tools.locks/ocamlformat")
+ (path "${dev_tool_lock_dir}")
  (repositories mock))
 
 (lock_dir

test/blackbox-tests/test-cases/pkg/ocamlformat/ocamlformat-avoid-conflict-with-project.t

@@ -29,7 +29,7 @@ Add a fake executable in the PATH
 
 Build the OCamlFormat binary dev-tool
   $ DUNE_CONFIG__LOCK_DEV_TOOL=enabled dune fmt --preview
-  Solution for dev-tools.locks/ocamlformat:
+  Solution for _build/.dev-tools.locks/ocamlformat:
   - ocamlformat.0.26.2
   File "dune", line 1, characters 0-0:
   Error: Files _build/default/dune and _build/default/.formatted/dune differ.

test/blackbox-tests/test-cases/pkg/ocamlformat/ocamlformat-avoid-taking-from-project-deps.t

@@ -45,7 +45,7 @@ Format using the dev-tools feature, it does not invoke the OCamlFormat binary fr
 the project dependencies (0.26.2) but instead builds and runs the OCamlFormat binary as a
 dev-tool (0.26.3).
   $ DUNE_CONFIG__LOCK_DEV_TOOL=enabled dune fmt
-  Solution for dev-tools.locks/ocamlformat:
+  Solution for _build/.dev-tools.locks/ocamlformat:
   - ocamlformat.0.26.3
   File "foo.ml", line 1, characters 0-0:
   Error: Files _build/default/foo.ml and _build/default/.formatted/foo.ml
@@ -59,7 +59,7 @@ Retry, without dev-tools feature and without cleaning. This time it uses the OCa
 binary from the project dependencies rather than the dev-tool. This exercises the
 behavior when OCamlFormat is installed simultaneously as both a dev-tool and as a
 regular package dependency.
-  $ rm -rf dev-tools.locks/ocamlformat
+  $ rm -r "${dev_tool_lock_dir}"
   $ dune fmt --preview
   File "foo.ml", line 1, characters 0-0:
   Error: Files _build/default/foo.ml and _build/default/.formatted/foo.ml