diff --git a/.cspell.json b/.cspell.json index bbc068e0..c1ec1ca1 100644 --- a/.cspell.json +++ b/.cspell.json @@ -90,6 +90,7 @@ "elif", "emeraldlake", "exitcode", + "filenamify", "githubaction", "githubactions", "gocyclo", @@ -149,6 +150,7 @@ "startswith", "staticcheck", "stmsg", + "tailscale", "testregex", "textfile", "tianocore", diff --git a/cmd/firmware-action/filesystem/filesystem.go b/cmd/firmware-action/filesystem/filesystem.go index 154f590e..e79d63b4 100644 --- a/cmd/firmware-action/filesystem/filesystem.go +++ b/cmd/firmware-action/filesystem/filesystem.go @@ -10,6 +10,8 @@ import ( "log/slog" "os" "path/filepath" + "regexp" + "strings" "time" "github.com/plus3it/gorecurcopy" @@ -244,6 +246,10 @@ func AnyFileNewerThan(path string, givenTime time.Time) (bool, error) { return nil }) if errors.Is(errMod, ErrFileModified) { + slog.Debug( + "Detected changes in files", + slog.Any("error", errMod), + ) return true, nil } return false, nil @@ -265,3 +271,34 @@ func AnyFileNewerThan(path string, givenTime time.Time) (bool, error) { // If path is neither file nor directory return false, err } + +func sanitizeAndTruncate(input string, length int) string { + // What characters are forbidden in Windows and Linux directory names? + // https://stackoverflow.com/a/31976060 + + // ASCII characters (printable and non-printable) + // https://en.wikipedia.org/wiki/Control_character#In_ASCII + charactersASCII := regexp.MustCompile(`[<>:"/\\|?*\x00-\x1F\x7F]`) + result := charactersASCII.ReplaceAllString(input, "_") + + // just for a good measure + betterAvoid := regexp.MustCompile(`[!{}()[\]^$+*?. -]`) + result = betterAvoid.ReplaceAllString(result, "_") + + // Remove invalid UTF-8 byte sequences + // - this includes unicode control characters + // https://www.compart.com/en/unicode/category/Cc + result = strings.ToValidUTF8(result, "_") + + if len(result) > length { + return string(result[:length]) + } + return result +} + +// Filenamify converts a string to a valid and safe filename +func Filenamify(name string, extension string) string { + // Maximum length (let's assume maximum of 255 bytes and let's assume 14 bytes for extension) + // https://serverfault.com/a/306726 + return sanitizeAndTruncate(name, 240) + "." + sanitizeAndTruncate(extension, 14) +} diff --git a/cmd/firmware-action/filesystem/filesystem_test.go b/cmd/firmware-action/filesystem/filesystem_test.go index c6a56372..3c233513 100644 --- a/cmd/firmware-action/filesystem/filesystem_test.go +++ b/cmd/firmware-action/filesystem/filesystem_test.go @@ -189,3 +189,127 @@ func TestAnyFileNewerThan(t *testing.T) { assert.NoError(t, err) assert.False(t, mod) } + +func TestFilenamify(t *testing.T) { + testCases := []struct { + name string + inputName string + inputExtension string + output string + }{ + { + name: "empty strings", + inputName: "", + inputExtension: "", + output: ".", + }, + { + name: "unicode control characters", + inputName: "foo\u0000bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foobar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo:bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo\"bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo/bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo\\bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo\bar", + inputExtension: "", + output: "foo_ar.", + }, + { + inputName: "foo|bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo?bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo*bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo/bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo!bar", + inputExtension: "", + output: "foo_bar.", + }, + { + inputName: "foo//bar", + inputExtension: "", + output: "foo__bar.", + }, + { + inputName: "//foo//bar//", + inputExtension: "", + output: "__foo__bar__.", + }, + { + inputName: "foo\\\\\\bar", + inputExtension: "", + output: "foo___bar.", + }, + { + inputName: "foo[*]bar", + inputExtension: "", + output: "foo___bar.", + }, + { + inputName: "foo bar", + inputExtension: "01234567890123456789", + output: "foo_bar.01234567890123", + }, + { + inputName: "foo\nbar", + inputExtension: "txt", + output: "foo_bar.txt", + }, + { + inputName: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789junk", + inputExtension: "txt", + output: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789.txt", + }, + } + for _, tc := range testCases { + t.Run(tc.inputName, func(t *testing.T) { + result := Filenamify(tc.inputName, tc.inputExtension) + assert.Equal(t, tc.output, result) + assert.LessOrEqual(t, len(result), 255) + }) + } +} diff --git a/cmd/firmware-action/go.mod b/cmd/firmware-action/go.mod index 66c070a2..9a11d523 100644 --- a/cmd/firmware-action/go.mod +++ b/cmd/firmware-action/go.mod @@ -1,6 +1,6 @@ module github.com/9elements/firmware-action/cmd/firmware-action -go 1.23.0 +go 1.23.1 toolchain go1.24.0 @@ -29,7 +29,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.6.0 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -47,14 +47,15 @@ require ( github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/sosodev/duration v1.3.1 // indirect github.com/vektah/gqlparser/v2 v2.5.22 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - go.opentelemetry.io/otel v1.32.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 // indirect @@ -63,13 +64,14 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 // indirect go.opentelemetry.io/otel/log v0.8.0 // indirect - go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect go.opentelemetry.io/otel/sdk v1.32.0 // indirect go.opentelemetry.io/otel/sdk/log v0.8.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect - go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.35.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect golang.org/x/net v0.35.0 // indirect golang.org/x/sync v0.11.0 // indirect golang.org/x/sys v0.30.0 // indirect diff --git a/cmd/firmware-action/go.sum b/cmd/firmware-action/go.sum index a9925b32..4b46566d 100644 --- a/cmd/firmware-action/go.sum +++ b/cmd/firmware-action/go.sum @@ -34,8 +34,9 @@ github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZ github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= @@ -111,8 +112,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/plus3it/gorecurcopy v0.0.1 h1:H7AgvM0N/uIo7o1PQRlewEGQ92BNr7DqbPy5lnR3uJI= github.com/plus3it/gorecurcopy v0.0.1/go.mod h1:NvVTm4RX68A1vQbHmHunDO4OtBLVroT6CrsiqAzNyJA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -136,8 +138,10 @@ github.com/vektah/gqlparser/v2 v2.5.22 h1:yaaeJ0fu+nv1vUMW0Hl+aS1eiv1vMfapBNjpff github.com/vektah/gqlparser/v2 v2.5.22/go.mod h1:xMl+ta8a5M1Yo1A1Iwt/k7gSpscwSnHZdw7tfhEGfTM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 h1:WzNab7hOOLzdDF/EoWCt4glhrbMPVMOO5JYTmpz36Ls= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0/go.mod h1:hKvJwTzJdp90Vh7p6q/9PAOd55dI6WA6sWj62a/JvSs= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 h1:S+LdBGiQXtJdowoJoQPEtI52syEP/JYBUpjO49EQhV8= @@ -154,16 +158,16 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI= go.opentelemetry.io/otel/log v0.8.0 h1:egZ8vV5atrUWUbnSsHn6vB8R21G2wrKqNiDt3iWertk= go.opentelemetry.io/otel/log v0.8.0/go.mod h1:M9qvDdUTRCopJcGRKg57+JSQ9LgLBrwwfC32epk5NX8= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= go.opentelemetry.io/otel/sdk/log v0.8.0 h1:zg7GUYXqxk1jnGF/dTdLPrK06xJdrXgqgFLnI4Crxvs= go.opentelemetry.io/otel/sdk/log v0.8.0/go.mod h1:50iXr0UVwQrYS45KbruFrEt4LvAdCaWWgIrsN3ZQggo= go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -171,8 +175,8 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= diff --git a/cmd/firmware-action/recipes/config.go b/cmd/firmware-action/recipes/config.go index b7f6eb08..2aa6459f 100644 --- a/cmd/firmware-action/recipes/config.go +++ b/cmd/firmware-action/recipes/config.go @@ -166,6 +166,8 @@ func (opts CommonOpts) GetOutputDir() string { return opts.OutputDir } +// ANCHOR: CommonOptsGetSources + // GetSources returns slice of paths to all sources which are used for build func (opts CommonOpts) GetSources() []string { sources := []string{} @@ -180,6 +182,8 @@ func (opts CommonOpts) GetSources() []string { return sources } +// ANCHOR_END: CommonOptsGetSources + // Config is for storing parsed configuration file type Config struct { // defined in coreboot.go @@ -294,9 +298,9 @@ type FirmwareModule interface { buildFirmware(ctx context.Context, client *dagger.Client) error } -// =========== -// Functions -// =========== +// ====================== +// Functions for Config +// ====================== // ValidateConfig is used to validate the configuration struct read out of JSON file func ValidateConfig(conf Config) error { @@ -357,7 +361,6 @@ func ReadConfig(filepath string) (*Config, error) { ) return nil, err } - contentStr := string(content) // Check if all environment variables are defined diff --git a/cmd/firmware-action/recipes/coreboot.go b/cmd/firmware-action/recipes/coreboot.go index 0621ccee..9eae413c 100644 --- a/cmd/firmware-action/recipes/coreboot.go +++ b/cmd/firmware-action/recipes/coreboot.go @@ -71,6 +71,8 @@ func (opts CorebootOpts) GetArtifacts() *[]container.Artifacts { return opts.CommonOpts.GetArtifacts() } +// ANCHOR: CorebootOptsGetSources + // GetSources returns slice of paths to all sources which are used for build func (opts CorebootOpts) GetSources() []string { sources := opts.CommonOpts.GetSources() @@ -109,6 +111,8 @@ func (opts CorebootOpts) GetSources() []string { return sources } +// ANCHOR_END: CorebootOptsGetSources + // ProcessBlobs is used to figure out blobs from provided data func (opts CorebootOpts) ProcessBlobs() ([]BlobDef, error) { blobs := []BlobDef{} diff --git a/cmd/firmware-action/recipes/recipes.go b/cmd/firmware-action/recipes/recipes.go index e2526285..38ba02a8 100644 --- a/cmd/firmware-action/recipes/recipes.go +++ b/cmd/firmware-action/recipes/recipes.go @@ -16,6 +16,7 @@ import ( "dagger.io/dagger" "github.com/9elements/firmware-action/cmd/firmware-action/filesystem" + "github.com/google/go-cmp/cmp" "github.com/heimdalr/dag" ) @@ -34,8 +35,13 @@ var ( var ( // ContainerWorkDir specifies directory in container used as work directory ContainerWorkDir = "/workdir" + // StatusDir is directory for temporary files generated by firmware-action to aid change detection + StatusDir = ".firmware-action" // TimestampsDir specifies directory for timestamps to detect changes in sources - TimestampsDir = ".firmware-action/timestamps" + TimestampsDir = filepath.Join(StatusDir, "timestamps") + // CompiledConfigsDir specifies directory for successfully compiled module configurations to detect changes in + // configuration + CompiledConfigsDir = filepath.Join(StatusDir, "configs") ) func forestAddVertex(forest *dag.DAG, key string, value FirmwareModule, dependencies [][]string) ([][]string, error) { @@ -179,14 +185,18 @@ func Execute(ctx context.Context, target string, config *Config) error { if err != nil { return err } + err = os.MkdirAll(CompiledConfigsDir, os.ModePerm) + if err != nil { + return err + } // Find requested target modules := config.AllModules() if _, ok := modules[target]; ok { - // Check if up-to-date - // Either returns time, or zero time and error - // zero time means there was no previous run - timestampFile := filepath.Join(TimestampsDir, fmt.Sprintf("%s.txt", target)) + // Check for any change in source files + // Either returns time, or zero time and error + // zero time means there was no previous run + timestampFile := filepath.Join(TimestampsDir, filesystem.Filenamify(target, "txt")) lastRun, _ := filesystem.LoadLastRunTime(timestampFile) sources := modules[target].GetSources() @@ -199,6 +209,33 @@ func Execute(ctx context.Context, target string, config *Config) error { } } + // Check for any change in configuration + // I did consider to save only the small struct related to each module, but it was + // proving to be far too much work. Instead we save the whole configuration file (for each module + // separately) and only compare the relevant modules between these two configurations + oldConfigPath := filepath.Join(CompiledConfigsDir, filesystem.Filenamify(target, "json")) + err = filesystem.CheckFileExists(oldConfigPath) + changedConfig := false + if errors.Is(err, os.ErrExist) { + oldConfig, err := ReadConfig(oldConfigPath) + // The config might be old / obsolete + // If the config is old / obsolete and no longer valid, it should just be ignored + // and it should be assumed that re-build is needed + if err != nil { + changedConfig = true + slog.Warn( + fmt.Sprintf("The configuration used for previous build, stored in '%s', is not valid and will be assumed obsolete", CompiledConfigsDir), + ) + } else { + oldModules := oldConfig.AllModules() + changedConfig = !cmp.Equal(modules[target], oldModules[target]) + } + } + slog.Debug("Changes were detected", + slog.Bool("sources", changesDetected), + slog.Bool("config", changedConfig), + ) + // Check if output directory already exist // We want to skip build if the output directory exists and is not empty // If it is empty, then just continue with the building @@ -206,12 +243,21 @@ func Execute(ctx context.Context, target string, config *Config) error { _, errExists := os.Stat(modules[target].GetOutputDir()) empty, _ := IsDirEmpty(modules[target].GetOutputDir()) if errExists == nil && !empty { - if changesDetected { + if changesDetected || changedConfig { // If any of the sources changed, we need to rebuild os.RemoveAll(modules[target].GetOutputDir()) } else { // Is already up-to-date slog.Warn(fmt.Sprintf("Target '%s' is up-to-date, skipping build", target)) + + // It is possible that the timestamp or old configuration files are missing + // for example user deleted them, CI does not cache them, ... + // but at the same time the output directory exists with all the artifacts + // If the 'override' is false, and if these files already exist, they will not + // be overridden + // We want these files to reflect last successful build, not last check + saveCheckpointTimeStamp(timestampFile, false) + saveCheckpointConfig(oldConfigPath, config, false) return ErrBuildUpToDate } } @@ -246,14 +292,38 @@ func Execute(ctx context.Context, target string, config *Config) error { // Build the module err = modules[target].buildFirmware(ctx, client) if err == nil { - // On success update the timestamp - _ = filesystem.SaveCurrentRunTime(timestampFile) + // On successful build, save timestamp and current configuration + saveCheckpointTimeStamp(timestampFile, true) + saveCheckpointConfig(oldConfigPath, config, true) } return err } return ErrTargetMissing } +func saveCheckpointTimeStamp(timestampFilePath string, override bool) { + // On success update the timestamp + err := filesystem.CheckFileExists(timestampFilePath) + if errors.Is(err, os.ErrNotExist) || override { + slog.Debug("Saving timestamp") + _ = filesystem.SaveCurrentRunTime(timestampFilePath) + } +} + +func saveCheckpointConfig(configPath string, config *Config, override bool) { + // On success update the old configuration + err := filesystem.CheckFileExists(configPath) + if errors.Is(err, os.ErrNotExist) || override { + slog.Debug("Saving copy of configuration file") + err = WriteConfig(configPath, config) + if err != nil { + slog.Warn("Failed to create a snapshot of configuration for detecting future changes", + slog.Any("error", err), + ) + } + } +} + // NormalizeArchitecture will translate various architecture strings into expected format func NormalizeArchitecture(arch string) string { archMap := map[string]string{ diff --git a/cmd/firmware-action/recipes/recipes_test.go b/cmd/firmware-action/recipes/recipes_test.go index 9f1fd21c..9375f7c7 100644 --- a/cmd/firmware-action/recipes/recipes_test.go +++ b/cmd/firmware-action/recipes/recipes_test.go @@ -4,9 +4,11 @@ package recipes import ( "context" "os" + "path/filepath" "testing" "dagger.io/dagger" + "github.com/9elements/firmware-action/cmd/firmware-action/filesystem" "github.com/stretchr/testify/assert" ) @@ -52,8 +54,8 @@ func TestExecuteSkipAndMissing(t *testing.T) { // Change current working directory pwd, err := os.Getwd() - defer os.Chdir(pwd) // nolint:errcheck assert.NoError(t, err) + defer os.Chdir(pwd) // nolint:errcheck tmpDir := t.TempDir() err = os.Chdir(tmpDir) assert.NoError(t, err) @@ -100,6 +102,91 @@ func TestExecuteSkipAndMissing(t *testing.T) { assert.ErrorIs(t, err, ErrDependencyOutputMissing) } +func TestExecuteUpToDate(t *testing.T) { + ctx := context.Background() + client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout)) + assert.NoError(t, err) + defer client.Close() + + // Change current working directory + pwd, err := os.Getwd() + assert.NoError(t, err) + defer os.Chdir(pwd) // nolint:errcheck + tmpDir := t.TempDir() + err = os.Chdir(tmpDir) + assert.NoError(t, err) + + // Create configuration + const target = "dummy" + const outputDir = "output-universal/" + const depends = "pre-dummy" + const outputDir2 = "output-universal2/" + const repopath = "fake-repo/" + config := Config{ + Universal: map[string]UniversalOpts{ + target: { + Depends: []string{depends}, + CommonOpts: CommonOpts{ + SdkURL: "whatever", + RepoPath: repopath, + OutputDir: outputDir, + ContainerInputDir: "inputs/", + ContainerOutputFiles: []string{ + "file.rom", + }, + }, + UniversalSpecific: UniversalSpecific{ + BuildCommands: []string{"false"}, + }, + }, + depends: { + CommonOpts: CommonOpts{ + SdkURL: "whatever", + RepoPath: repopath, + OutputDir: outputDir2, + ContainerInputDir: "inputs/", + ContainerOutputFiles: []string{ + "file.rom", + }, + }, + UniversalSpecific: UniversalSpecific{ + BuildCommands: []string{"false"}, + }, + }, + }, + } + + // Create directories needed by Execute() + err = os.MkdirAll(TimestampsDir, os.ModePerm) + assert.NoError(t, err) + err = os.MkdirAll(CompiledConfigsDir, os.ModePerm) + assert.NoError(t, err) + err = os.MkdirAll(repopath, os.ModePerm) + assert.NoError(t, err) + + // Create output dir and add a file to make it non-empty + err = os.MkdirAll(outputDir, os.ModePerm) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(outputDir, "file.rom"), []byte("test"), 0o666) + assert.NoError(t, err) + err = os.MkdirAll(outputDir2, os.ModePerm) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(outputDir2, "file.rom"), []byte("test"), 0o666) + assert.NoError(t, err) + + // Test 1: Skip due to up-to-date timestamp + timestampFile := filepath.Join(TimestampsDir, filesystem.Filenamify(target, "txt")) + saveCheckpointTimeStamp(timestampFile, true) + err = Execute(ctx, target, &config) + assert.ErrorIs(t, err, ErrBuildUpToDate) + + // Test 2: Skip due to up-to-date config + configFile := filepath.Join(CompiledConfigsDir, filesystem.Filenamify(target, "json")) + saveCheckpointConfig(configFile, &config, true) + err = Execute(ctx, target, &config) + assert.ErrorIs(t, err, ErrBuildUpToDate) +} + func executeDummy(_ context.Context, _ string, _ *Config) error { return nil } diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index fe130007..736be630 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -24,6 +24,7 @@ - [Build Docker container on the fly](firmware-action/build_dockerfile_on_the_fly.md) - [Interactive debugging](firmware-action/interactive.md) - [Offline usage](firmware-action/offline_usage.md) + - [Change detection](firmware-action/change_detection.md) - [Migration instructions]() - [Migration from v0.13.x to v0.14.0](firmware-action/migration/v0.13.x--v0.14.0/migrate.md) diff --git a/docs/src/firmware-action/change_detection.md b/docs/src/firmware-action/change_detection.md new file mode 100644 index 00000000..e82ff28e --- /dev/null +++ b/docs/src/firmware-action/change_detection.md @@ -0,0 +1,40 @@ +# Change detection + +firmware-action has basic detection of changes. + +Related temporary files to aid in change detection are stored in `.firmware-action/` directory, which is always created in current working directory. + +```admonish note +It is save to delete the `.firmware-action/` directory, but keep in mind that change detection depends on its existence. + +If `.firmware-action/` directory is deleted, it will be re-created on next firmware-action execution. + +It might be advantageous to also include this directory in caches / artifacts when in CI. Preserving these files might reduce run time in the CI. +``` + + +## Sources modification time + +When building a module, firmware-action checks recursively all sources for a given module. For all module types, list of sources include repository and all input files. + +```admonish example collapsible=true title="Code snippet: Common sources" +~~~golang +{{#include ../../../cmd/firmware-action/recipes/config.go:CommonOptsGetSources}} +~~~ +``` + +Each module type has then additional source files. For example `coreboot`, where list of sources also includes `defconfig` and all the blobs. +```admonish example collapsible=true title="Code snippet: Additional coreboot sources" +~~~golang +{{#include ../../../cmd/firmware-action/recipes/coreboot.go:CorebootOptsGetSources}} +~~~ +``` + +When a module is successfully built, a file containing time stamp is saved to `.firmware-action/timestamps/` directory. + +On next run, this file (if exists) is loaded with time stamp of last successful run. Then all sources are recursively checked for any file that was modified since the last successful run. If no file was modified since the loaded time stamp, module is considered up-to-date and build is skipped. If any of the files has newer modified time, module is re-built. + + +## Configuration file changes + +Firmware-action can also detect changes in the configuration file. For each module, on each successful build, it stores a copy of the configuration in `.firmware-action/configs/` directory. On next run, current configuration is compared to configuration of last successful build, and if the configuration for the specific module differs, module is re-built. diff --git a/docs/src/firmware-action/features.md b/docs/src/firmware-action/features.md index a150187c..b66dd523 100644 --- a/docs/src/firmware-action/features.md +++ b/docs/src/firmware-action/features.md @@ -5,4 +5,5 @@ - [Interactive mode](./interactive.md) - [Offline usage](./offline_usage.md) - [Recursive builds](./config.md#modules) +- [Change detection](./change_detection.md)