Enable GODEBUG=fips140=only tests in buildkite. (#302)

Enable unit tests with GODEBUG=fips140=only and -tags=requirefips in
buildkite. This is a basic check to make sure we don't accidently add
non-compliant crypto when in FIPS mode.
When these test are ran, the encrypted private key tests in
transport/tlscommon are skipped, the -tags=requirefips tests check that
an error is returned, however if fips140=only is set the test will
panic.
Add a new V2 file keystore that is used with requirefips. This
implementation uses NewGCMWithRandomNonce. File keystore support is
currently disabled in FIPS artifacts we produce.

Author: michel-laterman

.buildkite/pipeline.yml

@@ -23,6 +23,7 @@ steps:
     artifact_paths:
       - "junit-*.xml"
 
+    # Run unit tests with requirefips tag to validate functionality
   - label: ":linux: Test Linux FIPS"
     key: test-lin-fips
     command:
@@ -36,6 +37,19 @@ steps:
     artifact_paths:
       - "junit-*.xml"
 
+    # Run unit tests with requirefips tag and GODEBUG=fips140=only
+    # This is a check against accidentally adding crypto that breaks FIPS compliance.
+  - label: ":linux: Test Linux fips140=only"
+    key: test-lin-fipsonly
+    command:
+      - ".buildkite/scripts/test-fipsonly.sh"
+    agents:
+      image: golang:${GO_VERSION}
+      cpu: "8"
+      memory: "4G"
+    artifact_paths:
+      - "junit-*.xml"
+
   - label: ":windows: Test Windows"
     key: test-win
     command:

.buildkite/scripts/test-fipsonly.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+junitfile=$1 # filename for jnit annotation plugin
+
+set -euo pipefail
+
+echo "--- Pre install"
+source .buildkite/scripts/pre-install-command.sh
+go version
+add_bin_path
+with_go_junit_report
+
+echo "--- Go Test fips140=only"
+set +e
+GODEBUG=fips140=only go test -tags=integration,requirefips -json -race -v ./... > test-fips-report.json
+exit_code=$?
+set -e
+
+# Create Junit report for junit annotation plugin
+go-junit-report -parser gojson > "${junitfile:-junit-report-fips-linux.xml}" < test-fips-report.json
+exit $exit_code

go.mod

@@ -1,6 +1,6 @@
 module github.com/elastic/elastic-agent-libs
 
-go 1.24.1
+go 1.23
 
 require (
 	github.com/Microsoft/go-winio v0.5.2

keystore/file_keystore.go

@@ -19,8 +19,6 @@ package keystore
 
 import (
 	"bytes"
-	"crypto/aes"
-	"crypto/cipher"
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
@@ -45,9 +43,6 @@ const (
 	keyLength       = 32
 )
 
-// Version of the keystore format, will be added at the beginning of the file.
-var version = []byte("v1")
-
 // Packager defines a keystore that we can read the raw bytes and be packaged in an artifact.
 type Packager interface {
 	Package() ([]byte, error)
@@ -322,95 +317,6 @@ func (k *FileKeystore) load() error {
 	return jsonDecoder.Decode(&k.secrets)
 }
 
-// Encrypt the data payload using a derived keys and the AES-256-GCM algorithm.
-func (k *FileKeystore) encrypt(reader io.Reader) (io.Reader, error) {
-	// randomly generate the salt and the initialization vector, this information will be saved
-	// on disk in the file as part of the header
-	iv, err := randomBytes(iVLength)
-
-	if err != nil {
-		return nil, err
-	}
-
-	salt, err := randomBytes(saltLength)
-	if err != nil {
-		return nil, err
-	}
-
-	// Stretch the user provided key
-	password, _ := k.password.Get()
-	passwordBytes, err := k.hashPassword(string(password), salt)
-	if err != nil {
-		return nil, fmt.Errorf("could not hash password, error: %w", err)
-	}
-
-	// Select AES-256: because len(passwordBytes) == 32 bytes
-	block, err := aes.NewCipher(passwordBytes)
-	if err != nil {
-		return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %w", err)
-	}
-
-	aesgcm, err := cipher.NewGCM(block)
-	if err != nil {
-		return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %w", err)
-	}
-
-	data, err := io.ReadAll(reader)
-	if err != nil {
-		return nil, fmt.Errorf("could not read unencrypted data, error: %w", err)
-	}
-
-	encodedBytes := aesgcm.Seal(nil, iv, data, nil)
-
-	// Generate the payload with all the additional information required to decrypt the
-	// output format of the document: VERSION|SALT|IV|PAYLOAD
-	buf := bytes.NewBuffer(salt)
-	buf.Write(iv)
-	buf.Write(encodedBytes)
-
-	return buf, nil
-}
-
-// should receive an io.reader...
-func (k *FileKeystore) decrypt(reader io.Reader) (io.Reader, error) {
-	data, err := io.ReadAll(reader)
-	if err != nil {
-		return nil, fmt.Errorf("could not read all the data from the encrypted file, error: %w", err)
-	}
-
-	if len(data) < saltLength+iVLength+1 {
-		return nil, fmt.Errorf("missing information in the file for decrypting the keystore")
-	}
-
-	// extract the necessary information to decrypt the data from the data payload
-	salt := data[0:saltLength]
-	iv := data[saltLength : saltLength+iVLength]
-	encodedBytes := data[saltLength+iVLength:]
-
-	password, _ := k.password.Get()
-	passwordBytes, err := k.hashPassword(string(password), salt)
-	if err != nil {
-		return nil, fmt.Errorf("could not hash password, error: %w", err)
-	}
-
-	block, err := aes.NewCipher(passwordBytes)
-	if err != nil {
-		return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %w", err)
-	}
-
-	aesgcm, err := cipher.NewGCM(block)
-	if err != nil {
-		return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %w", err)
-	}
-
-	decodedBytes, err := aesgcm.Open(nil, iv, encodedBytes, nil)
-	if err != nil {
-		return nil, fmt.Errorf("could not decrypt keystore data: %w", err)
-	}
-
-	return bytes.NewReader(decodedBytes), nil
-}
-
 // checkPermission enforces permission on the keystore file itself, the file should have strict
 // permission (0600) and the keystore should refuses to start if its not the case.
 func (k *FileKeystore) checkPermissions(f string) error {

keystore/file_keystore_fips.go

@@ -0,0 +1,110 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build go1.24 && requirefips
+
+package keystore
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"fmt"
+	"io"
+)
+
+// Version of the keystore format, will be added at the beginning of the file.
+var version = []byte("v2")
+
+// Encrypt the data payload using a derived keys and the AES-256-GCM algorithm.
+func (k *FileKeystore) encrypt(reader io.Reader) (io.Reader, error) {
+	salt, err := randomBytes(saltLength)
+	if err != nil {
+		return nil, err
+	}
+
+	// Stretch the user provided key
+	password, _ := k.password.Get()
+	passwordBytes, err := k.hashPassword(string(password), salt)
+	if err != nil {
+		return nil, fmt.Errorf("could not hash password, error: %w", err)
+	}
+
+	// Select AES-256: because len(passwordBytes) == 32 bytes
+	block, err := aes.NewCipher(passwordBytes)
+	if err != nil {
+		return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %w", err)
+	}
+
+	aesgcm, err := cipher.NewGCMWithRandomNonce(block)
+	if err != nil {
+		return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %w", err)
+	}
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("could not read unencrypted data, error: %w", err)
+	}
+
+	encodedBytes := aesgcm.Seal(nil, nil, data, nil)
+
+	// Generate the payload with all the additional information required to decrypt the
+	// output format of the document: VERSION|SALT|PAYLOAD
+	buf := bytes.NewBuffer(salt)
+	buf.Write(encodedBytes)
+
+	return buf, nil
+}
+
+// should receive an io.reader...
+func (k *FileKeystore) decrypt(reader io.Reader) (io.Reader, error) {
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("could not read all the data from the encrypted file, error: %w", err)
+	}
+
+	if len(data) < saltLength+1 {
+		return nil, fmt.Errorf("missing information in the file for decrypting the keystore")
+	}
+
+	// extract the necessary information to decrypt the data from the data payload
+	salt := data[0:saltLength]
+	encodedBytes := data[saltLength:]
+
+	password, _ := k.password.Get()
+	passwordBytes, err := k.hashPassword(string(password), salt)
+	if err != nil {
+		return nil, fmt.Errorf("could not hash password, error: %w", err)
+	}
+
+	block, err := aes.NewCipher(passwordBytes)
+	if err != nil {
+		return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %w", err)
+	}
+
+	aesgcm, err := cipher.NewGCMWithRandomNonce(block)
+	if err != nil {
+		return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %w", err)
+	}
+
+	decodedBytes, err := aesgcm.Open(nil, nil, encodedBytes, nil)
+	if err != nil {
+		return nil, fmt.Errorf("could not decrypt keystore data: %w", err)
+	}
+
+	return bytes.NewReader(decodedBytes), nil
+}

keystore/file_keystore_fips_test.go

@@ -0,0 +1,64 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build go1.24 && requirefips
+
+package keystore
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestShouldRaiseAndErrorWhenVersionDontMatch(t *testing.T) {
+	temporaryPath := GetTemporaryKeystoreFile(t)
+	defer os.Remove(temporaryPath)
+
+	badVersion := `v1pqH8nRJNCuKLrAHwATQuHpdLcP84sATrxtKMWTvapZTRcoEODVJKf2dsHXiOhSMh1EFrJTikON2oF5wZv4IM37lkJ6wt79MCFaXDqlNxBQtIA9w6vaxWnbS+92rQqtka7WrzTxal1Pd3mcK0o+ow7EAJg553UvxBqA==`
+
+	f, err := os.OpenFile(temporaryPath, os.O_CREATE|os.O_WRONLY, 0600)
+	require.NoError(t, err)
+	_, _ = f.WriteString(badVersion)
+	err = f.Close()
+	require.NoError(t, err)
+
+	_, err = NewFileKeystoreWithPassword(temporaryPath, NewSecureString([]byte("")))
+	if assert.Error(t, err, "Expect version check error") {
+		assert.Equal(t, err, fmt.Errorf("keystore format doesn't match expected version: 'v2' got 'v1'"))
+	}
+}
+
+func TestOpensV2(t *testing.T) {
+	ks, err := NewFileKeystoreWithPassword(filepath.Join("testdata", "keystore.v2"), NewSecureString([]byte("")))
+	require.NoError(t, err)
+	ls, err := AsListingKeystore(ks)
+	require.NoError(t, err)
+	keys, err := ls.List()
+	require.NoError(t, err)
+	require.Len(t, keys, 1)
+	require.Equal(t, keys[0], "key")
+}
+
+func TestFailsToOpenV1(t *testing.T) {
+	_, err := NewFileKeystoreWithPassword(filepath.Join("testdata", "keystore.v1"), NewSecureString([]byte("")))
+	require.Error(t, err)
+}