Remove explicit OCaml transitive dependency files

Compute transitive module dependency closures directly from anonymous
ocamldep and ocamlobjinfo actions instead of materializing .all-deps
files. This removes the merge-file rule path for internal dependency
setup while preserving module-level cycle diagnostics.

As a side-effect, this fixes an ugly "Rule not found" error.

Signed-off-by: Rudi Grinberg <me@rgrinberg.com>

Author: rgrinberg

src/dune_rules/dep_rules.ml

@@ -2,13 +2,6 @@ open Import
 open Memo.O
 module Parallel_map = Memo.Make_parallel_map (Module_name.Unique.Map)
 
-let transitive_deps_contents modules =
-  List.map modules ~f:(fun m ->
-    (* TODO use object names *)
-    Modules.Sourced_module.to_module m |> Module.name |> Module_name.to_string)
-  |> String.concat ~sep:"\n"
-;;
-
 let ooi_deps
       ~vimpl
       ~sctx
@@ -17,11 +10,10 @@ let ooi_deps
       ~dune_version
       ~vlib_obj_map
       ~(ml_kind : Ml_kind.t)
-      ~for_
       (sourced_module : Modules.Sourced_module.t)
   =
   let m = Modules.Sourced_module.to_module sourced_module in
-  let* read =
+  let+ read =
     let unit =
       let cm_kind =
         match ml_kind with
@@ -42,7 +34,6 @@ let ooi_deps
       | [ x ] -> x
       | [] | _ :: _ -> assert false)
   in
-  let add_rule = Super_context.add_rule sctx ~dir in
   let read =
     Action_builder.memoize
       "ocamlobjinfo"
@@ -54,14 +45,6 @@ let ooi_deps
          then None
          else Module_name.Unique.Map.find vlib_obj_map dep))
   in
-  let+ () =
-    add_rule
-      (let target =
-         Obj_dir.Module.dep obj_dir ~for_ (Transitive (m, ml_kind)) |> Option.value_exn
-       in
-       Action_builder.map read ~f:transitive_deps_contents
-       |> Action_builder.write_file_dyn target)
-  in
   read
 ;;
 
@@ -72,77 +55,115 @@ let wrapped_compat_deps modules m =
   | None -> [ inner ]
 ;;
 
-let deps_of_module ~modules ~sandbox ~sctx ~dir ~obj_dir ~ml_kind ~for_ m =
-  match Module.kind m with
-  | Wrapped_compat ->
-    wrapped_compat_deps modules m |> Action_builder.return |> Memo.return
-  | _ ->
-    let+ deps = Ocamldep.deps_of ~sandbox ~modules ~sctx ~dir ~obj_dir ~ml_kind ~for_ m in
-    (match Modules.With_vlib.alias_for modules m with
-     | [] -> deps
-     | aliases ->
-       let open Action_builder.O in
-       let+ deps = deps in
-       aliases @ deps)
-;;
+type imported_vlib_deps =
+  { deps_of :
+      Modules.Sourced_module.t
+      -> ml_kind:Ml_kind.t
+      -> Module.t list Action_builder.t Memo.t
+  }
 
-let deps_of_vlib_module ~obj_dir ~vimpl ~dir ~sctx ~ml_kind ~for_ sourced_module =
-  match
-    let vlib = Vimpl.vlib vimpl in
-    Lib.Local.of_lib vlib
-  with
+let make_imported_vlib_deps ~obj_dir ~vimpl ~dir ~sctx ~sandbox =
+  let vlib = Vimpl.vlib vimpl in
+  match Lib.Local.of_lib vlib with
   | None ->
-    let+ deps =
-      let vlib_obj_map = Vimpl.vlib_obj_map vimpl in
-      let dune_version =
-        let impl = Vimpl.impl vimpl in
-        Dune_project.dune_version impl.project
+    let vlib_obj_map = Vimpl.vlib_obj_map vimpl in
+    let dune_version =
+      let impl = Vimpl.impl vimpl in
+      Dune_project.dune_version impl.project
+    in
+    let deps_of sourced_module ~ml_kind =
+      let+ deps =
+        ooi_deps
+          ~vimpl
+          ~sctx
+          ~dir
+          ~obj_dir
+          ~dune_version
+          ~vlib_obj_map
+          ~ml_kind
+          sourced_module
       in
-      ooi_deps
-        ~vimpl
-        ~sctx
-        ~dir
-        ~obj_dir
-        ~dune_version
-        ~vlib_obj_map
-        ~ml_kind
-        ~for_
-        sourced_module
+      Action_builder.map deps ~f:(fun deps ->
+        List.map deps ~f:Modules.Sourced_module.to_module)
     in
-    Action_builder.map deps ~f:(List.map ~f:Modules.Sourced_module.to_module)
+    { deps_of }
   | Some lib ->
     let vlib_obj_dir =
       let info = Lib.Local.info lib in
       Lib_info.obj_dir info
     in
-    let m = Modules.Sourced_module.to_module sourced_module in
-    let+ () =
-      let src =
-        Obj_dir.Module.dep vlib_obj_dir ~for_ (Transitive (m, ml_kind))
-        |> Option.value_exn
-        |> Path.build
-      in
-      let dst =
-        Obj_dir.Module.dep obj_dir ~for_ (Transitive (m, ml_kind)) |> Option.value_exn
-      in
-      Super_context.add_rule sctx ~dir (Action_builder.symlink ~src ~dst)
-    in
     let modules = Vimpl.vlib_modules vimpl |> Modules.With_vlib.modules in
-    Ocamldep.read_deps_of ~obj_dir:vlib_obj_dir ~modules ~ml_kind ~for_ m
+    let ocamldep =
+      Ocamldep.create
+        ~deps_of_imported_vlib:(fun _ ~ml_kind:_ -> None)
+        ~sandbox
+        ~modules
+        ~sctx
+        ~dir:(Obj_dir.dir vlib_obj_dir)
+        ~obj_dir:vlib_obj_dir
+        ()
+    in
+    let deps_of sourced_module ~ml_kind =
+      let m = Modules.Sourced_module.to_module sourced_module in
+      Ocamldep.deps_of ocamldep ~ml_kind m |> Memo.return
+    in
+    { deps_of }
+;;
+
+let deps_of_imported_vlib imported_vlib_deps m ~ml_kind =
+  let open Action_builder.O in
+  let* deps =
+    Action_builder.of_memo
+      (imported_vlib_deps.deps_of (Modules.Sourced_module.Imported_from_vlib m) ~ml_kind)
+  in
+  deps
+;;
+
+let make_ocamldep ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx =
+  let imported_vlib_deps =
+    match (Modules.With_vlib.split_by_lib modules).vlib with
+    | [] -> None
+    | _ :: _ ->
+      Some
+        (make_imported_vlib_deps
+           ~obj_dir
+           ~vimpl:(Virtual_rules.vimpl_exn impl)
+           ~dir
+           ~sctx
+           ~sandbox)
+  in
+  let deps_of_imported_vlib =
+    match imported_vlib_deps with
+    | None -> fun _ ~ml_kind:_ -> None
+    | Some deps -> fun m ~ml_kind -> Some (deps_of_imported_vlib deps m ~ml_kind)
+  in
+  ( Ocamldep.create ~deps_of_imported_vlib ~sandbox ~modules ~sctx ~dir ~obj_dir ()
+  , imported_vlib_deps )
+;;
+
+let deps_of_module ~modules ~ocamldep ~ml_kind m =
+  match Module.kind m with
+  | Wrapped_compat ->
+    wrapped_compat_deps modules m |> Action_builder.return |> Memo.return
+  | _ ->
+    let deps = Ocamldep.deps_of ocamldep ~ml_kind m in
+    Memo.return
+      (match Modules.With_vlib.alias_for modules m with
+       | [] -> deps
+       | aliases ->
+         let open Action_builder.O in
+         let+ deps = deps in
+         aliases @ deps)
 ;;
 
 (** Tests whether a set of modules is a singleton *)
 let has_single_file modules = Option.is_some @@ Modules.With_vlib.as_singleton modules
 
 let rec deps_of
-          ~obj_dir
           ~modules
-          ~sandbox
-          ~impl
-          ~dir
-          ~sctx
+          ~ocamldep
+          ~imported_vlib_deps
           ~ml_kind
-          ~for_
           (m : Modules.Sourced_module.t)
   =
   let is_alias_or_root =
@@ -157,31 +178,44 @@ let rec deps_of
   then Memo.return (Action_builder.return [])
   else (
     let skip_if_source_absent f sourced_module =
-      let m = Modules.Sourced_module.to_module m in
-      if Module.has m ~ml_kind
+      let module_ = Modules.Sourced_module.to_module sourced_module in
+      if Module.has module_ ~ml_kind
       then f sourced_module
       else Memo.return (Action_builder.return [])
     in
     match m with
     | Imported_from_vlib _ ->
-      let vimpl = Virtual_rules.vimpl_exn impl in
-      skip_if_source_absent
-        (deps_of_vlib_module ~obj_dir ~vimpl ~dir ~sctx ~ml_kind ~for_)
-        m
-    | Normal m ->
+      (match imported_vlib_deps with
+       | None -> Code_error.raise "imported vlib module without vlib deps" []
+       | Some imported_vlib_deps ->
+         skip_if_source_absent
+           (fun sourced_module -> imported_vlib_deps.deps_of sourced_module ~ml_kind)
+           m)
+    | Normal _ ->
       skip_if_source_absent
-        (deps_of_module ~modules ~sandbox ~sctx ~dir ~obj_dir ~ml_kind ~for_)
+        (fun sourced_module ->
+           Modules.Sourced_module.to_module sourced_module
+           |> deps_of_module ~modules ~ocamldep ~ml_kind)
         m
     | Impl_of_virtual_module impl_or_vlib ->
-      deps_of ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~ml_kind ~for_
+      deps_of ~modules ~ocamldep ~imported_vlib_deps ~ml_kind
       @@
       let m = Ml_kind.Dict.get impl_or_vlib ml_kind in
       (match ml_kind with
        | Intf -> Imported_from_vlib m
        | Impl -> Normal m))
 ;;
 
-let read_transitive_deps_of_module ~modules ~obj_dir ~ml_kind ~for_ unit =
+let read_transitive_deps_of_module
+      ~sandbox
+      ~sctx
+      ~modules
+      ~obj_dir
+      ~impl
+      ~dir
+      ~ml_kind
+      unit
+  =
   match Module.kind unit with
   | Root | Alias _ -> Action_builder.return []
   | Wrapped_compat -> wrapped_compat_deps modules unit |> Action_builder.return
@@ -190,7 +224,10 @@ let read_transitive_deps_of_module ~modules ~obj_dir ~ml_kind ~for_ unit =
     then Action_builder.return []
     else
       let open Action_builder.O in
-      let+ deps = Ocamldep.read_deps_of ~obj_dir ~modules ~ml_kind ~for_ unit in
+      let ocamldep, _imported_vlib_deps =
+        make_ocamldep ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx
+      in
+      let+ deps = Ocamldep.deps_of ocamldep ~ml_kind unit in
       (match Modules.With_vlib.alias_for modules unit with
        | [] -> deps
        | aliases -> aliases @ deps)
@@ -206,9 +243,10 @@ let read_immediate_deps_of ~sandbox ~sctx ~obj_dir ~modules ~ml_kind m =
     else Ocamldep.read_immediate_deps_of ~sandbox ~sctx ~obj_dir ~modules ~ml_kind m
 ;;
 
-let read_deps_of ~obj_dir ~modules ~ml_kind ~for_ m =
+let read_deps_of ~sandbox ~sctx ~obj_dir ~modules ~impl ~dir ~ml_kind m =
   if Module.has m ~ml_kind
-  then read_transitive_deps_of_module ~modules ~obj_dir ~ml_kind ~for_ m
+  then
+    read_transitive_deps_of_module ~sandbox ~sctx ~modules ~obj_dir ~impl ~dir ~ml_kind m
   else Action_builder.return []
 ;;
 
@@ -218,20 +256,26 @@ let dict_of_func_concurrently f =
   Ml_kind.Dict.make ~impl ~intf
 ;;
 
-let for_module ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~for_ module_ =
+let for_module ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~for_:_ module_ =
+  let ocamldep, imported_vlib_deps =
+    make_ocamldep ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx
+  in
   dict_of_func_concurrently
-    (deps_of ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~for_ (Normal module_))
+    (deps_of ~modules ~ocamldep ~imported_vlib_deps (Normal module_))
 ;;
 
-let rules ~obj_dir ~modules ~sandbox ~impl ~sctx ~dir ~for_ =
+let rules ~obj_dir ~modules ~sandbox ~impl ~sctx ~dir ~for_:_ =
   match Modules.With_vlib.as_singleton modules with
   | Some m -> Memo.return (Dep_graph.Ml_kind.dummy m)
   | None ->
+    let ocamldep, imported_vlib_deps =
+      make_ocamldep ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx
+    in
     dict_of_func_concurrently (fun ~ml_kind ->
       let+ per_module =
         Modules.With_vlib.obj_map modules
         |> Parallel_map.parallel_map ~f:(fun _obj_name m ->
-          deps_of ~obj_dir ~modules ~sandbox ~impl ~sctx ~dir ~ml_kind ~for_ m)
+          deps_of ~modules ~ocamldep ~imported_vlib_deps ~ml_kind m)
       in
       Dep_graph.make ~dir ~per_module)
     |> Memo.map ~f:(Dep_graph.Ml_kind.for_module_compilation ~modules)

src/dune_rules/dep_rules.mli

@@ -33,9 +33,12 @@ val read_immediate_deps_of
   -> Module.t list Action_builder.t
 
 val read_deps_of
-  :  obj_dir:Path.Build.t Obj_dir.t
+  :  sandbox:Sandbox_config.t
+  -> sctx:Super_context.t
+  -> obj_dir:Path.Build.t Obj_dir.t
   -> modules:Modules.With_vlib.t
+  -> impl:Virtual_rules.t
+  -> dir:Path.Build.t
   -> ml_kind:Ml_kind.t
-  -> for_:Compilation_mode.t
   -> Module.t
   -> Module.t list Action_builder.t

src/dune_rules/melange/melange_rules.ml

@@ -246,7 +246,15 @@ let make_same_lib_emission_deps =
       >>= function
       | true ->
         let* intf_deps =
-          Dep_rules.read_deps_of ~obj_dir ~modules ~ml_kind:Intf module_ ~for_
+          Dep_rules.read_deps_of
+            ~sandbox
+            ~sctx
+            ~obj_dir
+            ~modules
+            ~impl:Virtual_rules.no_implements
+            ~dir:(Obj_dir.dir obj_dir)
+            ~ml_kind:Intf
+            module_
         in
         (* Cross-module optimization follows implementation artifacts, but the
          initial reachability also comes from the emitted module's interface
@@ -256,10 +264,10 @@ let make_same_lib_emission_deps =
         |> Action_builder.map ~f:(deps_of_xopt_closure ~obj_dir)
       | false ->
         (* Emission reads same-library implementation artifacts recursively.
-           The generic [.impl.all-deps] files collapse transitive edges through
-           interfaces when a dependency has an [.mli], which is fine for
-           compilation but insufficient for JS emission. Follow the
-           implementation dependency graph directly instead. *)
+           Compilation dependencies collapse transitive edges through interfaces
+           when a dependency has an [.mli], which is insufficient for JS
+           emission. Follow the implementation dependency graph directly
+           instead. *)
         Dep_graph.top_closed_implementations dep_graph [ module_ ]
         |> Action_builder.map ~f:(deps_of_impl_closure ~obj_dir)
 ;;