* Adds support for private Bazel deps

Author: njlr

buckaroo/Constants.fs

@@ -18,6 +18,9 @@ let LockFileName = "buckaroo.lock.toml"
 [<Literal>]
 let BuckarooMacrosFileName = "buckaroo_macros.bzl"
 
+[<Literal>]
+let BuckarooDefsFileName = "defs.bzl"
+
 [<Literal>]
 let BuckarooDepsFileName = "BUCKAROO_DEPS"
 

buckaroo/InstallCommand.fs

@@ -2,6 +2,7 @@ module Buckaroo.InstallCommand
 
 open System
 open System.IO
+open FSharp.Control
 open FSharpx.Control
 open Buckaroo
 open Buckaroo.BuckConfig
@@ -19,35 +20,7 @@ let private fetchManifestFromLock (lock : Lock) (sourceExplorer : ISourceExplore
   return! sourceExplorer.FetchManifest location
 }
 
-let private fetchDependencyTargets (lock : Lock) (sourceExplorer : ISourceExplorer) (manifest : Manifest) = async {
-  let! targetIdentifiers =
-    manifest.Dependencies
-    |> Seq.map (fun d -> async {
-      let! targets =
-        match d.Targets with
-        | Some targets -> async {
-            return targets |> List.toSeq
-          }
-        | None -> async {
-            let! manifest = fetchManifestFromLock lock sourceExplorer d.Package
-            return manifest.Targets |> Set.toSeq
-          }
-
-      return
-        targets
-        |> Seq.map (fun target ->
-          { Package = d.Package; Target = target }
-        )
-    })
-    |> Async.Parallel
-
-  return
-    targetIdentifiers
-    |> Seq.collect id
-    |> Seq.toList
-}
-
-let private buckarooMacros =
+let private buckarooBuckMacros =
   [
     "def buckaroo_cell(package): ";
     "  cell = native.read_config('buckaroo', package, '').strip()";
@@ -91,11 +64,24 @@ let rec computeCellIdentifier (parents : PackageIdentifier list) (package : Pack
   | head::tail ->
     (computeCellIdentifier tail head) + "." + (computeCellIdentifier [] package)
 
-let rec private combinePaths xs =
-  match xs with
-  | [ x ] -> x
-  | x::xs -> Path.Combine (x, combinePaths xs)
-  | [] -> ""
+let private packageWorkspaceName (parents : PackageIdentifier list) (package : PackageIdentifier) =
+  let s =
+    seq {
+      yield! parents
+      yield package
+    }
+    |> Seq.map PackageIdentifier.show
+    |> String.concat "_"
+
+  "buckaroo_" +
+  (
+    s
+    |> Strings.replaceAll [ "/"; "."; "-" ] "_"
+  ) + "_" +
+  (
+    Hashing.sha256 s
+    |> Strings.truncate 8
+  )
 
 let rec packageInstallPath (parents : PackageIdentifier list) (package : PackageIdentifier) =
   match parents with
@@ -104,7 +90,7 @@ let rec packageInstallPath (parents : PackageIdentifier list) (package : Package
       match package with
       | PackageIdentifier.GitHub x -> ("github", x.Owner, x.Project)
       | PackageIdentifier.BitBucket x -> ("bitbucket", x.Owner, x.Project)
-      | PackageIdentifier.GitLab x -> ("gitlab", combinePaths x.Groups, x.Project)
+      | PackageIdentifier.GitLab x -> ("gitlab", Paths.combineAll x.Groups, x.Project)
       | PackageIdentifier.Adhoc x -> ("adhoc", x.Owner, x.Project)
     String.Join ("/", [ Constants.PackagesDirectory; prefix; owner; project ])
   | head::tail ->
@@ -345,65 +331,40 @@ let private generateBuckConfig (sourceExplorer : ISourceExplorer) (parents : Pac
   return buckarooConfig |> BuckConfig.render
 }
 
-let private workspaceName (package : PackageIdentifier) =
-  "buckaroo_" + (PackageIdentifier.show package)
-  |> Strings.replace "-" "_"
-  |> Strings.replace "." "_"
-  |> Strings.replace "/" "_"
-
-let private dataBzl (packages : Map<PackageIdentifier, LockedPackage>) =
+let private generateBazelDefs (targets : Set<TargetIdentifier>) =
   seq {
-    yield "packages = {"
+    yield "def buckaroo_deps():"
+
+    yield "  return ["
     yield!
-      packages
-      |> Map.toSeq
-      |> Seq.collect (fun (packageIdentifier, lockedPackage) -> seq {
-        yield "  \"" + (PackageIdentifier.show packageIdentifier) + "\": {"
-        yield "    \"workspace\": \"" + (workspaceName packageIdentifier) + "\","
-        yield "    \"path\": \"" + (packageInstallPath [] packageIdentifier) + "\", "
-        yield "  }, "
+      targets
+      |> Seq.collect (fun t -> seq {
+        let parents, package = t.PackagePath
+        yield "    \"@" + (packageWorkspaceName parents package) + (Target.show t.Target) + "\","
       })
-    yield "}"
+
+    yield "  ]"
+
     yield ""
   }
   |> String.concat "\n"
 
-let private defsBzl = """
-load("//:data.bzl", "packages")
-
-# Utility function to get the name of the Bazel WORKSPACE at which a package is installed.
-def buckaroo_workspace(package_name):
-  package = packages.get(package_name)
-  if package:
-    return package.get("workspace")
-  else:
-    fail("The package " + package_name + " could not be found. ")
-
-# Utility function to be used in the user's WORKSPACE to wire-up each package.
-def buckaroo_setup():
-  for package in packages.values():
-    native.local_repository(
-      name = package.get("workspace"),
-      path = package.get("path"),
-    )
-"""
-
 let rec private installBuckPackages (context : Tasks.TaskContext) (root : string) (parents : PackageIdentifier list) (packages : Map<PackageIdentifier, LockedPackage>) = async {
   // Prepare workspace
   do! Files.mkdirp root
   do! Files.touch (Path.Combine(root, ".buckconfig"))
 
   if File.Exists (Path.Combine(root, Constants.BuckarooMacrosFileName)) |> not
   then
-    do! Files.writeFile (Path.Combine(root, Constants.BuckarooMacrosFileName)) buckarooMacros
+    do! Files.writeFile (Path.Combine(root, Constants.BuckarooMacrosFileName)) buckarooBuckMacros
 
   // Install packages
   for (package, lockedPackage) in packages |> Map.toSeq do
     let installPath =
       Path.Combine(root, packageInstallPath [] package)
       |> Paths.normalize
 
-    let childParents = (parents @ [ package ])
+    let childParents = parents @ [ package ]
 
     // Install child package sources
     do! installPackageSources context installPath lockedPackage.Location lockedPackage.Versions
@@ -419,37 +380,113 @@ let rec private installBuckPackages (context : Tasks.TaskContext) (root : string
     do! Files.writeFile (Path.Combine(installPath, ".buckconfig.d", ".buckconfig.buckaroo")) buckarooConfig
 }
 
+let rec private computeNestedPackages (parents : PackageIdentifier list) packages =
+  packages
+  |> Map.toSeq
+  |> Seq.collect (fun (k, v) -> seq {
+    yield (parents, k)
+    yield! computeNestedPackages (parents @ [ k ]) v.PrivatePackages
+  })
+
+let private generateTopLevelBazelDefs (sourceExplorer : ISourceExplorer) (lock : Lock) = async {
+  return
+    seq {
+      yield "def buckaroo_setup():"
+
+      // We need to create a "local_repository" for every package and its private packages
+      yield!
+        lock.Packages
+        |> computeNestedPackages []
+        |> Seq.collect (fun (parents, package) -> seq {
+          yield "  native.local_repository("
+          yield "    name = \"" + (packageWorkspaceName parents package) + "\","
+          yield "    path = \"" + (packageInstallPath parents package) + "\""
+          yield "  )"
+          yield ""
+        })
+
+      yield "  return None"
+      yield ""
+      yield generateBazelDefs lock.Dependencies
+    }
+    |> String.concat "\n"
+}
+
 let rec private installBazelPackages (context : Tasks.TaskContext) (root : string) (parents : PackageIdentifier list) (packages : Map<PackageIdentifier, LockedPackage>) = async {
-  // Prepare workspace
-  do! Files.mkdirp root
-  do! Files.touch (Path.Combine (root, "WORKSPACE"))
+  do! Files.touch (Paths.combine root "WORKSPACE")
 
   // Install packages
   for (package, lockedPackage) in packages |> Map.toSeq do
     let installPath =
-      Path.Combine(root, packageInstallPath [] package)
+      Paths.combine root (packageInstallPath [] package)
       |> Paths.normalize
 
-    let childParents = (parents @ [ package ])
+    let childParents = parents @ [ package ]
 
     // Install child package sources
     do! installPackageSources context installPath lockedPackage.Location lockedPackage.Versions
 
-    // Install child's child (recurse)
+    // Install child's children (recurse)
     do! installBazelPackages context installPath childParents lockedPackage.PrivatePackages
 
-    // Write macros
-    do! Files.writeFile (Path.Combine ("buckaroo", "data.bzl")) (dataBzl packages)
-    do! Files.writeFile (Path.Combine ("buckaroo", "defs.bzl")) defsBzl
-}
+    // Write buckaroo/defs.bzl
+    do! Files.mkdirp (Paths.combineAll [ installPath; Constants.PackagesDirectory ])
+
+    do! Files.touch (Paths.combineAll [ installPath; Constants.PackagesDirectory; "BUILD.bazel" ])
+
+    let! manifest = context.SourceExplorer.FetchManifest (lockedPackage.Location, lockedPackage.Versions)
+
+    let! targets =
+      asyncSeq {
+        let xs =
+          manifest.Dependencies
+          |> Seq.map (fun d -> (parents, d))
+          |> Seq.append (
+            manifest.PrivateDependencies
+            |> Seq.map (fun d -> (parents @ [ package ], d))
+          )
+
+        for parents, dependency in xs do
+          match dependency.Targets with
+          | Some targets ->
+            yield!
+              targets
+              |> Seq.map (fun target ->
+                {
+                  PackagePath = parents, dependency.Package
+                  Target = target
+                }
+              )
+              |> AsyncSeq.ofSeq
+          | None ->
+            // If the manifest does not specify any targets then
+            // we need to take the defaults listed in the package manifest.
+            let lockedPackage =
+              packages
+              |> Map.find dependency.Package // TODO: Better error if this fails. Indicates an invalid lock-file?
+
+            let! manifest =
+              context.SourceExplorer.FetchManifest (lockedPackage.Location, lockedPackage.Versions)
+
+            yield!
+              manifest.Targets
+              |> Seq.map (fun target ->
+                {
+                  PackagePath = parents, dependency.Package
+                  Target = target
+                }
+              )
+              |> AsyncSeq.ofSeq
+      }
+      |> AsyncSeq.toListAsync
+      |> Async.map Set.ofSeq
+
+    let buckarooDefs =
+      generateBazelDefs
+        targets
 
-let rec private computeNestedPackages (parents : PackageIdentifier list) packages =
-  packages
-  |> Map.toSeq
-  |> Seq.collect (fun (k, v) -> seq {
-    yield (parents, k)
-    yield! computeNestedPackages (parents @ [ k ]) v.PrivatePackages
-  })
+    do! Files.writeFile (Path.Combine(installPath, Constants.PackagesDirectory, Constants.BuckarooDefsFileName)) buckarooDefs
+}
 
 let writeTopLevelBuckFiles (context : Tasks.TaskContext) (root : string) (lock : Lock) = async {
   let nestedPackages =
@@ -476,7 +513,7 @@ let writeTopLevelBuckFiles (context : Tasks.TaskContext) (root : string) (lock :
         "dependencies",
         (
           lock.Dependencies
-          |> Seq.map (fun d -> (computeCellIdentifier [] d.Package) + (Target.show d.Target))
+          |> Seq.map (fun d -> (computeCellIdentifier (fst d.PackagePath) (snd d.PackagePath)) + (Target.show d.Target))
           |> String.concat " "
           |> INIString
         )
@@ -493,6 +530,15 @@ let writeTopLevelBuckFiles (context : Tasks.TaskContext) (root : string) (lock :
   do! Files.writeFile (Path.Combine (root, ".buckconfig.d", ".buckconfig.buckaroo")) (BuckConfig.render config)
 }
 
+let writeTopLevelBazelFiles context lock = async {
+  do! Files.mkdirp (Paths.combine "." Constants.PackagesDirectory)
+  do! Files.touch (Paths.combineAll [ "."; Constants.PackagesDirectory; "BUILD.bazel" ])
+
+  let! bazelDefs = generateTopLevelBazelDefs context.SourceExplorer lock
+
+  do! Files.writeFile (Paths.combineAll [ "."; Constants.PackagesDirectory; Constants.BuckarooDefsFileName ]) bazelDefs
+}
+
 let task (context : Tasks.TaskContext) = async {
   let logger = createLogger context.Console None
 
@@ -506,15 +552,11 @@ let task (context : Tasks.TaskContext) = async {
 
     match context.BuildSystem with
     | Buck ->
-      do! installBuckPackages context "." [] lock.Packages
       do! writeTopLevelBuckFiles context "." lock
+      do! installBuckPackages context "." [] lock.Packages
     | Bazel ->
-      do! Files.mkdirp "buckaroo"
-      do! Files.touch (Paths.combine "buckaroo" "WORKSPACE")
-      do! Files.touch (Paths.combine "buckaroo" "BUILD")
+      do! writeTopLevelBazelFiles context lock
       do! installBazelPackages context "." [] lock.Packages
-      // failwith "Not implemented"
-      // TODO
 
     logger.Success "The packages folder is now up-to-date. "
 

buckaroo/Lock.fs

@@ -146,7 +146,7 @@ module Lock =
         |> Map.tryFind p
         |> Option.map (fun (rv, _) ->
           rv.Manifest.Targets
-          |> Seq.map (fun t -> { Package = p; Target = t })
+          |> Seq.map (fun t -> { PackagePath = [], p; Target = t })
         )
         |> Option.defaultValue Seq.empty
       )
@@ -187,7 +187,7 @@ module Lock =
       lock.Dependencies
       |> Seq.map(fun x ->
         "[[dependency]]\n" +
-        "package = \"" + (PackageIdentifier.show x.Package) + "\"\n" +
+        "package = \"" + (PackageIdentifier.show (snd x.PackagePath)) + "\"\n" +
         "target = \"" + (Target.show x.Target) + "\"\n\n"
       )
       |> String.concat ""
@@ -415,7 +415,11 @@ module Lock =
       |> Result.bind (Toml.asString >> Result.mapError Toml.TomlError.show)
       |> Result.bind Target.parse
 
-    return { Package = package; Target = target }
+    return
+      {
+        PackagePath = [], package
+        Target = target
+      }
   }
 
   let parse (content : string) : Result<Lock, string> = result {

buckaroo/PackageLocation.fs

@@ -67,4 +67,4 @@ module PackageLocation =
     | PackageLocation.Git g -> Set [Version.Git (GitVersion.Revision g.Revision)]
     | PackageLocation.GitLab g -> Set [Version.Git (GitVersion.Revision g.Revision)]
     | PackageLocation.BitBucket g -> Set [Version.Git (GitVersion.Revision g.Revision)]
-    | _ -> Set.empty
+    | _ -> Set.empty