[8.19] (backport #8124) Generate NOTICE-fips.txt file (#8136)

* Generate `NOTICE-fips.txt` file (#8124)

* Amending code to generate NOTICE-fips.txt file alongside NOTICE.txt file

* Adding generated NOTICE-fips.txt file

* Adding CHANGELOG entry

* Update dependabot workflow to also generate NOTICE-fips.txt file

* Update notice target doc in Makefile

* Update notice documentation in README

* Cleanup

* Override bundled NOTICE.txt with contents of NOTICE-fips.txt in FIPS-capable artifacts

* Adding PR link to CHANGELOG entry

* Adding NOTICE check to mage TestPackages workflow

* Making comment consistent

* Consolidate NOTICE check/update/commit steps

(cherry picked from commit 2648f22)

* Regenerating FIPS-notice.txt

---------

Co-authored-by: Shaunak Kashyap <[email protected]>

Author: mergify[bot]

.github/workflows/post-dependabot.yml

@@ -1,7 +1,7 @@
 # Follow-on actions relating to dependabot PRs. In elastic/elastic-agent, any changes to
 # dependencies contained in go.mod requires the change to be reflected in the
-# NOTICE.txt file. When dependabot creates a branch for a go_modules change this
-# will update the NOTICE.txt file for that change.
+# NOTICE.txt and NOTICE-fips.txt files. When dependabot creates a branch for a go_modules
+# change this will update the NOTICE.txt and NOTICE-fips files for that change.
 name: post-dependabot
 
 on:
@@ -26,26 +26,26 @@ jobs:
         with:
           install-only: true
 
-      - name: update NOTICE.txt
+      - name: update NOTICE.txt and NOTICE-fips.txt
         run: make notice
 
-      - name: check for modified NOTICE.txt
+      - name: check for modified NOTICE.txt or NOTICE-fips.txt
         id: notice-check
         run: |
-          if git diff --quiet HEAD -- NOTICE.txt; then
+          if git diff --quiet HEAD -- NOTICE*.txt; then
             echo "modified=false" >> $GITHUB_OUTPUT
           else
             echo "modified=true" >> $GITHUB_OUTPUT
           fi
 
-      - name: commit NOTICE.txt
+      - name: commit NOTICE.txt and/or NOTICE-fips.txt
         if: steps.notice-check.outputs.modified == 'true'
         run: |
           git config --global user.name 'dependabot[bot]'
           git config --global user.email 'dependabot[bot]@users.noreply.github.com'
-          git add NOTICE.txt
+          git add NOTICE*.txt
           git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
-          git commit -m "Update NOTICE.txt"
+          git commit -m "Update NOTICE.txt and/or NOTICE-fips.txt"
           git push
 
       - name: update otel README.md

Makefile

@@ -18,7 +18,7 @@ help: Makefile
 	@printf "Variables:\n"
 	@grep -E "^[A-Za-z0-9_]*\?=" $< | awk 'BEGIN {FS = "\\?="}; { printf "  \033[36m%-25s\033[0m  Default values: %s\n", $$1, $$2}'
 
-## notice : Generates the NOTICE file.
+## notice : Generates the NOTICE.txt and NOTICE-fips.txt files.
 .PHONY: notice
 notice:
 	@mage notice

NOTICE-fips.txt

@@ -258,12 +258,6 @@ rules implemented on our `Makefile` as well as CI will use the
 locally before submitting any PRs to have a quicker feedback instead
 of waiting for a CI failure.
 
-### Generating the `NOTICE.txt` when updating/adding dependencies
-To do so, just run `make notice`, this is also part of the `make
+### Generating the `NOTICE.txt` and `NOTICE-fips.txt` when updating/adding dependencies
+To do so, just run `mage notice`, this is also part of the `make
 check-ci` and is the same check our CI will do.
-
-At some point we will migrate it to mage (see discussion on
-https://github.com/elastic/elastic-agent/pull/1108 and on
-https://github.com/elastic/elastic-agent/issues/1107). However until
-we have the mage automation sorted out, it has been removed to avoid
-confusion.

README.md

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: other
+
+# Change summary; a 80ish characters long description of the change.
+summary: A new NOTICE file, `NOTICE-fips.txt` is included in FIPS-capable Agent distributions.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/8124
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1746736812-fips-notice.yaml

@@ -267,6 +267,7 @@ func TestPackages(options ...TestPackagesOption) error {
 	}
 
 	args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*"))
+	args = append(args, "--source-root", MustExpand("{{.PWD}}"))
 
 	if out, err := goTest(args...); err != nil {
 		if mg.Verbose() {

dev-tools/mage/pkg.go

@@ -15,6 +15,8 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/elastic/elastic-agent/dev-tools/notice"
+
 	"github.com/magefile/mage/sh"
 )
 
@@ -28,17 +30,29 @@ func runCommand(cmd string, args ...string) error {
 	return nil
 }
 
-// Notice Generates NOTICE.txt.
+// Notice Generates NOTICE.txt and NOTICE-fips.txt
 func Notice() (err error) {
-	fmt.Println("Generating NOTICE")
+	if err := generateNotice(notice.NoticeFilename); err != nil {
+		return fmt.Errorf("failed to generate %s: %w", notice.NoticeFilename, err)
+	}
+	if err := generateNotice(notice.FIPSNoticeFilename, "requirefips"); err != nil {
+		return fmt.Errorf("failed to generate %s: %w", notice.FIPSNoticeFilename, err)
+	}
+	return nil
+}
+
+// generateNotice generates a generateNotice file with the name outputFilename.
+// see getDependentModules for use of additionalTags.
+func generateNotice(outputFilename string, additionalTags ...string) error {
+	fmt.Printf("Generating %s...\n", outputFilename)
 	if err := runCommand("go", "mod", "tidy"); err != nil {
 		return err
 	}
 	if err := runCommand("go", "mod", "download"); err != nil {
 		return err
 	}
 
-	modules, err := getDependentModules()
+	modules, err := getDependentModules(additionalTags...)
 	if err != nil {
 		return fmt.Errorf("unable to fetch list of dependent modules: %w", err)
 	}
@@ -52,7 +66,7 @@ func Notice() (err error) {
 	// -rules dev-tools/notice/rules.json \
 	// -overrides dev-tools/notice/overrides.json \
 	// -noticeTemplate dev-tools/notice/NOTICE.txt.tmpl \
-	// -noticeOut NOTICE.txt \
+	// -noticeOut {outputFilename} \
 	// -depsOut ""
 	listCmdArgs := []string{"list", "-m", "-json"}
 	listCmdArgs = append(listCmdArgs, modules...)
@@ -62,7 +76,7 @@ func Notice() (err error) {
 		"-rules", "dev-tools/notice/rules.json",
 		"-overrides", "dev-tools/notice/overrides.json",
 		"-noticeTemplate", "dev-tools/notice/NOTICE.txt.tmpl",
-		"-noticeOut", "NOTICE.txt",
+		"-noticeOut", outputFilename,
 		"-depsOut", "")
 
 	fmt.Printf(">> %s | %s\n", strings.Join(listCmd.Args, " "), strings.Join(licDetectCmd.Args, " "))
@@ -96,22 +110,21 @@ func Notice() (err error) {
 		return err
 	}
 
-	// cat dev-tools/notice/NOTICE.txt.append >> NOTICE.txt
-	fmt.Printf(">> %s\n", "cat dev-tools/notice/NOTICE.txt.append >> NOTICE.txt")
+	// cat dev-tools/notice/NOTICE.txt.append >> {outputFilename}
 	const (
-		infn  = "dev-tools/notice/NOTICE.txt.append"
-		outfn = "NOTICE.txt"
+		infn = "dev-tools/notice/NOTICE.txt.append"
 	)
+	fmt.Printf(">> cat %s >> %s\n", infn, outputFilename)
 
 	f, err := os.Open(infn)
 	if err != nil {
 		return fmt.Errorf("failed to open file %s: %w", infn, err)
 	}
 	defer f.Close()
 
-	out, err := os.OpenFile(outfn, os.O_WRONLY|os.O_APPEND, 0644)
+	out, err := os.OpenFile(outputFilename, os.O_WRONLY|os.O_APPEND, 0644)
 	if err != nil {
-		return fmt.Errorf("failed to open file %s: %w", outfn, err)
+		return fmt.Errorf("failed to open file %s: %w", outputFilename, err)
 	}
 
 	defer func() {
@@ -122,23 +135,23 @@ func Notice() (err error) {
 	}()
 
 	if _, err := io.Copy(out, f); err != nil {
-		return fmt.Errorf("failed to append file %s: %w", outfn, err)
+		return fmt.Errorf("failed to append file %s: %w", outputFilename, err)
 	}
 
-	// dos2unix NOTICE.txt
-	fmt.Printf(">> %s\n", "dos2unix NOTICE.txt")
+	// dos2unix {outputFilename}
+	fmt.Printf(">> dos2unix %s\n", outputFilename)
 
-	content, err := os.ReadFile(outfn)
+	content, err := os.ReadFile(outputFilename)
 	if err != nil {
-		return fmt.Errorf("failed to read entire file %s: %w", outfn, err)
+		return fmt.Errorf("failed to read entire file %s: %w", outputFilename, err)
 	}
 
 	// Convert Windows-style line endings to Unix-style
 	newContent := strings.ReplaceAll(string(content), "\r\n", "\n")
 
-	err = os.WriteFile(outfn, []byte(newContent), 0644)
+	err = os.WriteFile(outputFilename, []byte(newContent), 0644)
 	if err != nil {
-		return fmt.Errorf("failed to rewrite file using Unix-style line endings %s: %w", outfn, err)
+		return fmt.Errorf("failed to rewrite file using Unix-style line endings %s: %w", outputFilename, err)
 	}
 
 	return nil

dev-tools/mage/target/common/notice.go

@@ -0,0 +1,10 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package notice
+
+const (
+	NoticeFilename     = "NOTICE.txt"
+	FIPSNoticeFilename = "NOTICE-fips.txt"
+)

dev-tools/notice/notice.go

@@ -435,6 +435,10 @@ shared:
     files:
       <<: *agent_binary_files
       <<: *agent_unpacked_components_files
+      NOTICE.txt:
+        source: '{{ repo.RootDir }}/NOTICE-fips.txt'
+        mode: 0644
+
 
   - &agent_darwin_binary_spec
     <<: *common

dev-tools/packaging/packages.yml

@@ -36,6 +36,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/elastic-agent/dev-tools/mage"
+	"github.com/elastic/elastic-agent/dev-tools/notice"
 	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
 )
 
@@ -65,6 +66,7 @@ var (
 
 var (
 	files             = flag.String("files", "../build/distributions/*", "filepath glob containing package files")
+	sourceRoot        = flag.String("source-root", "../", "path to root directory of Agent repository")
 	modules           = flag.Bool("modules", false, "check modules folder contents")
 	minModules        = flag.Int("min-modules", 4, "minimum number of modules to expect in modules folder")
 	modulesd          = flag.Bool("modules.d", false, "check modules.d folder contents")
@@ -190,6 +192,7 @@ func checkTar(t *testing.T, file string, fipsCheck bool) {
 	require.NoErrorf(t, err, "error extracting archive %q", file)
 
 	t.Run("check_manifest_file", testManifestFile(tempExtractionPath, fipsCheck))
+	t.Run("check_notice_file", testNoticeFile(tempExtractionPath, fipsCheck))
 
 	checkSha512PackageHash(t, file)
 
@@ -220,20 +223,31 @@ func checkZip(t *testing.T, file string) {
 	require.NoErrorf(t, err, "error extracting archive %q", file)
 
 	t.Run("check_manifest_file", testManifestFile(tempExtractionPath, false))
+	t.Run("check_notice_file", testNoticeFile(tempExtractionPath, false))
 
 	checkSha512PackageHash(t, file)
 }
 
 func testManifestFile(agentPackageRootDir string, checkFips bool) func(t *testing.T) {
 	return func(t *testing.T) {
-		dirEntries, err := os.ReadDir(agentPackageRootDir)
-		require.NoErrorf(t, err, "error listing extraction dir %q", agentPackageRootDir)
-		require.Lenf(t, dirEntries, 1, "archive should contain a single directory: found %v", dirEntries)
-		containingDir := dirEntries[0].Name()
-		checkManifestFileContents(t, filepath.Join(agentPackageRootDir, containingDir))
+		checkManifestFileContents(t, getExtractedPackageDir(agentPackageRootDir, t))
 	}
 }
 
+func testNoticeFile(agentPackageRootDir string, checkFips bool) func(t *testing.T) {
+	return func(t *testing.T) {
+		checkNoticeFileContents(t, getExtractedPackageDir(agentPackageRootDir, t), checkFips)
+	}
+}
+
+func getExtractedPackageDir(agentPackageRootDir string, t *testing.T) string {
+	dirEntries, err := os.ReadDir(agentPackageRootDir)
+	require.NoErrorf(t, err, "error listing extraction dir %q", agentPackageRootDir)
+	require.Lenf(t, dirEntries, 1, "archive should contain a single directory: found %v", dirEntries)
+
+	return filepath.Join(agentPackageRootDir, dirEntries[0].Name())
+}
+
 func checkManifestFileContents(t *testing.T, extractedPackageDir string) {
 	t.Log("Checking file manifest.yaml")
 	m := parseManifest(t, extractedPackageDir)
@@ -275,6 +289,47 @@ func parseManifest(t *testing.T, dir string) v1.PackageManifest {
 	return *m
 }
 
+func checkNoticeFileContents(t *testing.T, extractedPackageDir string, checkFips bool) {
+	t.Logf("Checking package file NOTICE.txt; checkFips = %t", checkFips)
+
+	// Hash the source NOTICE file
+	sourceNoticeFile := filepath.Join(*sourceRoot, notice.NoticeFilename)
+	if checkFips {
+		sourceNoticeFile = filepath.Join(*sourceRoot, notice.FIPSNoticeFilename)
+	}
+	sourceNoticeFile, err := filepath.Abs(sourceNoticeFile)
+	require.NoError(t, err)
+
+	sourceNoticeFileHash, err := fileHash(sourceNoticeFile)
+	require.NoError(t, err)
+
+	// Hash the NOTICE file in the package
+	packageNoticeFile := filepath.Join(extractedPackageDir, "NOTICE.txt")
+	packageNoticeFileHash, err := fileHash(packageNoticeFile)
+	require.NoError(t, err)
+
+	// Compare the two hashes; they should be equal
+	require.Equalf(
+		t, sourceNoticeFileHash, packageNoticeFileHash,
+		"Contents of NOTICE.txt file in package are not the same as contents of %s", sourceNoticeFile,
+	)
+}
+
+func fileHash(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	h := sha512.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return nil, err
+	}
+
+	return h.Sum(nil), nil
+}
+
 const (
 	npcapLicense    = `Dependency : Npcap \(https://nmap.org/npcap/\)`
 	libpcapLicense  = `Dependency : Libpcap \(http://www.tcpdump.org/\)`
@@ -621,11 +676,7 @@ func checkDockerUser(t *testing.T, p *packageFile, info *dockerInfo, expectRoot
 }
 
 func checkFIPS(t *testing.T, agentPackageRootDir string) {
-	dirEntries, err := os.ReadDir(agentPackageRootDir)
-	require.NoErrorf(t, err, "error listing extraction dir %q", agentPackageRootDir)
-	require.Lenf(t, dirEntries, 1, "archive should contain a single directory: found %v", dirEntries)
-
-	extractedPackageDir := filepath.Join(agentPackageRootDir, dirEntries[0].Name())
+	extractedPackageDir := getExtractedPackageDir(agentPackageRootDir, t)
 	t.Logf("Checking agent binary in %q for FIPS compliance", extractedPackageDir)
 	m := parseManifest(t, extractedPackageDir)
 	versionedHome := m.Package.VersionedHome