Generate exports for stdlib (#4244)

**What type of PR is this?**

Feature

**What does this PR do? Why is it needed?**
This PR adds exports to the standard library which is useful for the
gopackagesdriver to avoid parsing files and various other tools that use
https://pkg.go.dev/golang.org/x/tools/go/packages.

**Which issues(s) does this PR fix?**
#3653 

**Other notes for review**
Basically a reimplementation of this PR:
#3653 but with the comment
addressed about configuring it with `//go/config`. One additional thing
added is the exports files are part of the `GoSDK` provider so that
other custom rules can also gain access to the exported files.

Author: jgautier-dd

BUILD.bazel

@@ -127,6 +127,7 @@ go_config(
         "//go/private:is_compilation_mode_dbg": "//go/private:always_true",
         "//conditions:default": "//go/config:debug",
     }),
+    export_stdlib = "//go/config:export_stdlib",
     gc_goopts = "//go/config:gc_goopts",
     gc_linkopts = "//go/config:gc_linkopts",
     gotags = "//go/config:tags",

go/config/BUILD.bazel

@@ -90,3 +90,9 @@ filegroup(
     name = "empty",
     visibility = ["//visibility:public"],
 )
+
+bool_flag(
+    name = "export_stdlib",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)

go/modes.rst

@@ -44,47 +44,54 @@ The build settings below are defined in the package
 ``@io_bazel_rules_go//go/config``. They can all be set on the command line
 or using `Bazel configuration transitions`_.
 
-+-------------------+----------------+-----------------------------------------+
-| **Name**          | **Type**       | **Default value**                       |
-+-------------------+---------------------+------------------------------------+
-| :param:`static`   | :type:`bool`        | :value:`false`                     |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| **Name**               | **Type**            | **Default value**             |
++------------------------+---------------------+-------------------------------+
+| :param:`static`        | :type:`bool`        | :value:`false`                |
++------------------------+---------------------+-------------------------------+
 | Statically links the target binary. May not always work since parts of the   |
 | standard library and other C dependencies won't tolerate static linking.     |
 | Works best with ``pure`` set as well.                                        |
-+-------------------+---------------------+------------------------------------+
-| :param:`race`     | :type:`bool`        | :value:`false`                     |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| :param:`race`          | :type:`bool`        | :value:`false`                |
++------------------------+---------------------+-------------------------------+
 | Instruments the binary for race detection. Programs will panic when a data   |
 | race is detected. Requires cgo. Mutually exclusive with ``msan``.            |
-+-------------------+---------------------+------------------------------------+
-| :param:`msan`     | :type:`bool`        | :value:`false`                     |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| :param:`msan`          | :type:`bool`        | :value:`false`                |
++------------------------+---------------------+-------------------------------+
 | Instruments the binary for memory sanitization. Requires cgo. Mutually       |
 | exclusive with ``race``.                                                     |
-+-------------------+---------------------+------------------------------------+
-| :param:`pure`     | :type:`bool`        | :value:`false`                     |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| :param:`pure`          | :type:`bool`        | :value:`false`                |
++------------------------+---------------------+-------------------------------+
 | Disables cgo, even when a C/C++ toolchain is configured (similar to setting  |
 | ``CGO_ENABLED=0``). Packages that contain cgo code may still be built, but   |
 | the cgo code will be filtered out, and the ``cgo`` build tag will be false.  |
-+-------------------+---------------------+------------------------------------+
-| :param:`debug`    | :type:`bool`        | :value:`false`                     |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| :param:`debug`         | :type:`bool`        | :value:`false`                |
++------------------------+---------------------+-------------------------------+
 | Includes debugging information in compiled packages (using the ``-N`` and    |
 | ``-l`` flags). This is always true with ``-c dbg``.                          |
-+-------------------+---------------------+------------------------------------+
-| :param:`gotags`   | :type:`string_list` | :value:`[]`                        |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| :param:`gotags`        | :type:`string_list` | :value:`[]`                   |
++------------------------+---------------------+-------------------------------+
 | Controls which build tags are enabled when evaluating build constraints in   |
 | source files. Useful for conditional compilation.                            |
-+-------------------+---------------------+------------------------------------+
-| :param:`linkmode` | :type:`string`      | :value:`"normal"`                  |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| :param:`linkmode`      | :type:`string`      | :value:`"normal"`             |
++------------------------+---------------------+-------------------------------+
 | Determines how the Go binary is built and linked. Similar to ``-buildmode``. |
 | Must be one of ``"normal"``, ``"shared"``, ``"pie"``, ``"plugin"``,          |
 | ``"c-shared"``, ``"c-archive"``.                                             |
-+-------------------+---------------------+------------------------------------+
++------------------------+---------------------+-------------------------------+
+| :param:`export_stdlib` | :type:`bool`        | :value:`false`                |
++------------------------+---------------------+-------------------------------+
+| This controls whether exports for the stdlib are generated by rules_go.      |
+| This is useful for running tools like golintci-lint via GOPACKAGESDRIVER     |
+| but adds time to the initial build. Leave false unless you want to use       |
+| golangci-lint or another tool that relies on GOPACKAGESDRIVER.               |
++------------------------+---------------------+-------------------------------+
 
 Platforms
 ---------

go/private/actions/stdlib.bzl

@@ -71,6 +71,8 @@ def _build_stdlib_list_json(go):
     args.add("-sdk", sdk.root_file.dirname)
     args.add("-out", out)
     args.add("-cache", cache_dir.path)
+    if go.export_stdlib:
+        args.add("-export", go.export_stdlib)
 
     inputs_direct = [sdk.go]
     inputs_transitive = [sdk.headers, sdk.srcs, sdk.libs, sdk.tools]
@@ -86,7 +88,7 @@ def _build_stdlib_list_json(go):
         env = _build_env(go),
         toolchain = GO_TOOLCHAIN_LABEL,
     )
-    return out
+    return out, cache_dir
 
 def _build_env(go):
     env = go.env
@@ -115,8 +117,10 @@ def _build_env(go):
     return env
 
 def _sdk_stdlib(go):
+    list_json, cache_dir = _build_stdlib_list_json(go)
     return GoStdLib(
-        _list_json = _build_stdlib_list_json(go),
+        _list_json = list_json,
+        cache_dir = depset([cache_dir]),
         libs = go.sdk.libs,
         root_file = go.sdk.root_file,
     )
@@ -164,8 +168,10 @@ def _build_stdlib(go):
         toolchain = GO_TOOLCHAIN_LABEL,
         execution_requirements = SUPPORTS_PATH_MAPPING_REQUIREMENT,
     )
+    list_json, cache_dir = _build_stdlib_list_json(go)
     return GoStdLib(
-        _list_json = _build_stdlib_list_json(go),
+        _list_json = list_json,
         libs = depset([pkg]),
+        cache_dir = depset([cache_dir]),
         root_file = pkg,
     )

go/private/context.bzl

@@ -453,6 +453,7 @@ default_go_config_info = GoConfigInfo(
     amd64 = None,
     arm = None,
     pgoprofile = None,
+    export_stdlib = False,
 )
 
 def go_context(
@@ -625,6 +626,7 @@ def go_context(
         coverdata = go_context_info.coverdata if go_context_info else None,
         coverage_enabled = ctx.configuration.coverage_enabled,
         coverage_instrumented = ctx.coverage_instrumented(),
+        export_stdlib = go_config_info.export_stdlib,
         env = env,
         # Path mapping can't map the values of environment variables, so we pass GOROOT to the action
         # via an argument instead in builder_args. We need to drop it from the environment to get cache
@@ -1001,6 +1003,7 @@ def _go_config_impl(ctx):
         amd64 = ctx.attr.amd64,
         arm = ctx.attr.arm,
         pgoprofile = pgoprofile,
+        export_stdlib = ctx.attr.export_stdlib[BuildSettingInfo].value,
     )
     validate_mode(go_config_info)
 
@@ -1057,6 +1060,10 @@ go_config = rule(
             mandatory = True,
             allow_files = True,
         ),
+        "export_stdlib": attr.label(
+            mandatory = False,
+            providers = [BuildSettingInfo],
+        ),
     },
     provides = [GoConfigInfo],
     doc = """Collects information about build settings in the current

go/providers.rst

@@ -389,9 +389,13 @@ from GoSDK_, or it may be another library compiled for the target mode.
 +--------------------------------+-----------------------------------------------------------------+
 | :param:`root_file`             | :type:`File`                                                    |
 +--------------------------------+-----------------------------------------------------------------+
-| A file or directory in the standard library root directory. Used to determine ``GOROOT``.                     |
+| A file or directory in the standard library root directory. Used to determine ``GOROOT``.        |
 +--------------------------------+-----------------------------------------------------------------+
 | :param:`libs`                  | :type:`list of File`                                            |
 +--------------------------------+-----------------------------------------------------------------+
 | .a files for the standard library, built for the target platform.                                |
 +--------------------------------+-----------------------------------------------------------------+
+| :param:`cache_dir`             | :type:`list of File`                                            |
++--------------------------------+-----------------------------------------------------------------+
+| GOCACHE directory for the stdlib after running `go list`.                                        |
++--------------------------------+-----------------------------------------------------------------+

go/tools/builders/stdliblist.go

@@ -166,7 +166,7 @@ func flatPackageForStd(cloneBase string, pkg *goListPackage, pathReplaceFn func(
 		ID:              stdlibPackageID(pkg.ImportPath),
 		Name:            pkg.Name,
 		PkgPath:         pkg.ImportPath,
-		ExportFile:      outputBasePath(cloneBase, pkg.Target),
+		ExportFile:      pathReplaceFn(pkg.Export),
 		Imports:         map[string]string{},
 		Standard:        pkg.Standard,
 		GoFiles:         goFiles,
@@ -203,6 +203,8 @@ func stdliblist(args []string) error {
 	goenv := envFlags(flags)
 	out := flags.String("out", "", "Path to output go list json")
 	cachePath := flags.String("cache", "", "Path to use for GOCACHE")
+	export := flags.Bool("export", false, "Should -export be passed to go list")
+
 	if err := flags.Parse(args); err != nil {
 		return err
 	}
@@ -276,6 +278,10 @@ func stdliblist(args []string) error {
 		listArgs = append(listArgs, "-compiled=true")
 	}
 
+	if *export {
+		listArgs = append(listArgs, "-export")
+	}
+
 	listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo")
 
 	jsonFile, err := os.Create(*out)

go/tools/builders/stdliblist_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 )
 
-func Test_stdliblist(t *testing.T) {
+func Test_stdliblist_noexport(t *testing.T) {
 	testDir := t.TempDir()
 	outJSON := filepath.Join(testDir, "out.json")
 
@@ -36,8 +36,8 @@ func Test_stdliblist(t *testing.T) {
 		if !strings.HasPrefix(result.ID, "@//stdlib:") {
 			t.Errorf("ID should be prefixed with @//stdlib: :%v", result)
 		}
-		if !strings.HasPrefix(result.ExportFile, "__BAZEL_OUTPUT_BASE__") {
-			t.Errorf("export file should be prefixed with __BAZEL_OUTPUT_BASE__ :%v", result)
+		if result.ExportFile != "" {
+			t.Errorf("ExportsFile should be empty when disabled but got: %v", result)
 		}
 		for _, gofile := range result.GoFiles {
 			// The SDK runfiles are prefixed with __BAZEL_OUTPUT_BASE__/../go_sdk, which is cleaned.
@@ -47,3 +47,48 @@ func Test_stdliblist(t *testing.T) {
 		}
 	}
 }
+
+func Test_stdliblist_export(t *testing.T) {
+	testDir := t.TempDir()
+	outJSON := filepath.Join(testDir, "out.json")
+	test_args := []string{
+		fmt.Sprintf("-out=%s", outJSON),
+		"-sdk=../go_sdk",
+		"-export",
+	}
+	// Disable CGO otherwise, this takes forever to build.
+	t.Setenv("CGO_ENABLED", "0")
+
+	if err := stdliblist(test_args); err != nil {
+		t.Errorf("calling stdliblist got err: %v", err)
+	}
+	f, err := os.Open(outJSON)
+	if err != nil {
+		t.Errorf("cannot open output json: %v", err)
+	}
+	defer func() { _ = f.Close() }()
+	decoder := json.NewDecoder(f)
+	anyExportSet := false
+	for decoder.More() {
+		var result *flatPackage
+		if err := decoder.Decode(&result); err != nil {
+			t.Errorf("unable to decode output json: %v\n", err)
+		}
+
+		if !strings.HasPrefix(result.ID, "@//stdlib:") {
+			t.Errorf("ID should be prefixed with @//stdlib: :%v", result)
+		}
+		if result.ExportFile != "" {
+			anyExportSet = true
+		}
+		for _, gofile := range result.GoFiles {
+			// The SDK runfiles are prefixed with __BAZEL_OUTPUT_BASE__/../go_sdk, which is cleaned.
+			if !strings.HasPrefix(gofile, "go_sdk/") {
+				t.Errorf("all go files should be prefixed with go_sdk/ :%v", result)
+			}
+		}
+	}
+	if !anyExportSet {
+		t.Error("At least one export file should be set when -export is set.")
+	}
+}