diff --git a/internal/go/base/error_notunix.go b/internal/go/base/error_notunix.go new file mode 100644 index 000000000000..c7780fa300a8 --- /dev/null +++ b/internal/go/base/error_notunix.go @@ -0,0 +1,12 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !unix + +package base + +func IsETXTBSY(err error) bool { + // syscall.ETXTBSY is only meaningful on Unix platforms. + return false +} diff --git a/internal/go/base/error_unix.go b/internal/go/base/error_unix.go new file mode 100644 index 000000000000..2dcd75e5f368 --- /dev/null +++ b/internal/go/base/error_unix.go @@ -0,0 +1,16 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package base + +import ( + "errors" + "syscall" +) + +func IsETXTBSY(err error) bool { + return errors.Is(err, syscall.ETXTBSY) +} diff --git a/internal/go/base/readme.md b/internal/go/base/readme.md new file mode 100644 index 000000000000..93afe9f28332 --- /dev/null +++ b/internal/go/base/readme.md @@ -0,0 +1,10 @@ +# quoted + +Extracted from `go/src/cmd/go/internal/base/` (related to `cache`). + +Only the function `IsETXTBSY` is extracted. + +## History + +- https://github.com/golangci/golangci-lint/pull/5576 + - sync go1.24.1 diff --git a/internal/go/cache/cache.go b/internal/go/cache/cache.go index 67c3e48f91a6..c514613dce76 100644 --- a/internal/go/cache/cache.go +++ b/internal/go/cache/cache.go @@ -23,9 +23,10 @@ import ( "time" "github.com/rogpeppe/go-internal/lockedfile" + "github.com/rogpeppe/go-internal/robustio" + "github.com/golangci/golangci-lint/v2/internal/go/base" "github.com/golangci/golangci-lint/v2/internal/go/mmap" - "github.com/golangci/golangci-lint/v2/internal/go/robustio" ) // An ActionID is a cache action key, the hash of a complete description of a @@ -41,8 +42,8 @@ type Cache interface { // Get returns the cache entry for the provided ActionID. // On miss, the error type should be of type *entryNotFoundError. // - // After a success call to Get, OutputFile(Entry.OutputID) must - // exist on disk for until Close is called (at the end of the process). + // After a successful call to Get, OutputFile(Entry.OutputID) must + // exist on disk until Close is called (at the end of the process). Get(ActionID) (Entry, error) // Put adds an item to the cache. @@ -53,14 +54,14 @@ type Cache interface { // As a special case, if the ReadSeeker is of type noVerifyReadSeeker, // the verification from GODEBUG=goverifycache=1 is skipped. // - // After a success call to Get, OutputFile(Entry.OutputID) must - // exist on disk for until Close is called (at the end of the process). + // After a successful call to Put, OutputFile(OutputID) must + // exist on disk until Close is called (at the end of the process). Put(ActionID, io.ReadSeeker) (_ OutputID, size int64, _ error) // Close is called at the end of the go process. Implementations can do // cache cleanup work at this phase, or wait for and report any errors from - // background cleanup work started earlier. Any cache trimming should in one - // process should not violate cause the invariants of this interface to be + // background cleanup work started earlier. Any cache trimming in one + // process should not cause the invariants of this interface to be // violated in another process. Namely, a cache trim from one process should // not delete an ObjectID from disk that was recently Get or Put from // another process. As a rule of thumb, don't trim things used in the last @@ -105,7 +106,7 @@ func Open(dir string) (*DiskCache, error) { } for i := 0; i < 256; i++ { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) - if err := os.MkdirAll(name, 0744); err != nil { + if err := os.MkdirAll(name, 0o777); err != nil { return nil, err } } @@ -161,13 +162,13 @@ var errVerifyMode = errors.New("gocacheverify=1") var DebugTest = false // func init() { initEnv() } - +// // var ( // gocacheverify = godebug.New("gocacheverify") // gocachehash = godebug.New("gocachehash") // gocachetest = godebug.New("gocachetest") // ) - +// // func initEnv() { // if gocacheverify.Value() == "1" { // gocacheverify.IncNonDefault() @@ -258,10 +259,7 @@ func (c *DiskCache) get(id ActionID) (Entry, error) { return missing(errors.New("negative timestamp")) } - err = c.used(c.fileName(id, "a")) - if err != nil { - return Entry{}, fmt.Errorf("failed to mark %s as used: %w", c.fileName(id, "a"), err) - } + c.markUsed(c.fileName(id, "a")) return Entry{buf, size, time.Unix(0, tm)}, nil } @@ -305,25 +303,35 @@ func GetBytes(c Cache, id ActionID) ([]byte, Entry, error) { // GetMmap looks up the action ID in the cache and returns // the corresponding output bytes. // GetMmap should only be used for data that can be expected to fit in memory. -func GetMmap(c Cache, id ActionID) ([]byte, Entry, error) { +func GetMmap(c Cache, id ActionID) ([]byte, Entry, bool, error) { entry, err := c.Get(id) if err != nil { - return nil, entry, err + return nil, entry, false, err } - md, err := mmap.Mmap(c.OutputFile(entry.OutputID)) + md, opened, err := mmap.Mmap(c.OutputFile(entry.OutputID)) if err != nil { - return nil, Entry{}, err + return nil, Entry{}, opened, err } if int64(len(md.Data)) != entry.Size { - return nil, Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} + return nil, Entry{}, true, &entryNotFoundError{Err: errors.New("file incomplete")} } - return md.Data, entry, nil + return md.Data, entry, true, nil } // OutputFile returns the name of the cache file storing output with the given OutputID. func (c *DiskCache) OutputFile(out OutputID) string { file := c.fileName(out, "d") - c.used(file) + isDir := c.markUsed(file) + if isDir { // => cached executable + entries, err := os.ReadDir(file) + if err != nil { + return fmt.Sprintf("DO NOT USE - missing binary cache entry: %v", err) + } + if len(entries) != 1 { + return "DO NOT USE - invalid binary cache entry" + } + return filepath.Join(file, entries[0].Name()) + } return file } @@ -345,7 +353,7 @@ const ( trimLimit = 5 * 24 * time.Hour ) -// used makes a best-effort attempt to update mtime on file, +// markUsed makes a best-effort attempt to update mtime on file, // so that mtime reflects cache access time. // // Because the reflection only needs to be approximate, @@ -354,25 +362,17 @@ const ( // mtime is more than an hour old. This heuristic eliminates // nearly all of the mtime updates that would otherwise happen, // while still keeping the mtimes useful for cache trimming. -func (c *DiskCache) used(file string) error { +// +// markUsed reports whether the file is a directory (an executable cache entry). +func (c *DiskCache) markUsed(file string) (isDir bool) { info, err := os.Stat(file) - if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval { - return nil - } - if err != nil { - if os.IsNotExist(err) { - return &entryNotFoundError{Err: err} - } - return &entryNotFoundError{Err: fmt.Errorf("failed to stat file %s: %w", file, err)} + return false } - - err = os.Chtimes(file, c.now(), c.now()) - if err != nil { - return fmt.Errorf("failed to change time of file %s: %w", file, err) + if now := c.now(); now.Sub(info.ModTime()) >= mtimeInterval { + os.Chtimes(file, now, now) } - - return nil + return info.IsDir() } func (c *DiskCache) Close() error { return c.Trim() } @@ -410,7 +410,7 @@ func (c *DiskCache) Trim() error { // cache will appear older than it is, and we'll trim it again next time. var b bytes.Buffer fmt.Fprintf(&b, "%d", now.Unix()) - if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0666); err != nil { + if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0o666); err != nil { return err } @@ -439,6 +439,10 @@ func (c *DiskCache) trimSubdir(subdir string, cutoff time.Time) { entry := filepath.Join(subdir, name) info, err := os.Stat(entry) if err == nil && info.ModTime().Before(cutoff) { + if info.IsDir() { // executable cache entry + os.RemoveAll(entry) + continue + } os.Remove(entry) } } @@ -471,7 +475,7 @@ func (c *DiskCache) putIndexEntry(id ActionID, out OutputID, size int64, allowVe // Copy file to cache directory. mode := os.O_WRONLY | os.O_CREATE - f, err := os.OpenFile(file, mode, 0666) + f, err := os.OpenFile(file, mode, 0o666) if err != nil { return err } @@ -517,7 +521,21 @@ func (c *DiskCache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error if isNoVerify { file = wrapper.ReadSeeker } - return c.put(id, file, !isNoVerify) + return c.put(id, "", file, !isNoVerify) +} + +// PutExecutable is used to store the output as the output for the action ID into a +// file with the given base name, with the executable mode bit set. +// It may read file twice. The content of file must not change between the two passes. +func (c *DiskCache) PutExecutable(id ActionID, name string, file io.ReadSeeker) (OutputID, int64, error) { + if name == "" { + panic("PutExecutable called without a name") + } + wrapper, isNoVerify := file.(noVerifyReadSeeker) + if isNoVerify { + file = wrapper.ReadSeeker + } + return c.put(id, name, file, !isNoVerify) } // PutNoVerify is like Put but disables the verify check @@ -528,7 +546,7 @@ func PutNoVerify(c Cache, id ActionID, file io.ReadSeeker) (OutputID, int64, err return c.Put(id, noVerifyReadSeeker{file}) } -func (c *DiskCache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) { +func (c *DiskCache) put(id ActionID, executableName string, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) { // Compute output ID. h := sha256.New() if _, err := file.Seek(0, 0); err != nil { @@ -542,7 +560,11 @@ func (c *DiskCache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (Outp h.Sum(out[:0]) // Copy to cached output file (if not already present). - if err := c.copyFile(file, out, size); err != nil { + fileMode := fs.FileMode(0o666) + if executableName != "" { + fileMode = 0o777 + } + if err := c.copyFile(file, executableName, out, size, fileMode); err != nil { return out, size, err } @@ -558,9 +580,33 @@ func PutBytes(c Cache, id ActionID, data []byte) error { // copyFile copies file into the cache, expecting it to have the given // output ID and size, if that file is not present already. -func (c *DiskCache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { - name := c.fileName(out, "d") +func (c *DiskCache) copyFile(file io.ReadSeeker, executableName string, out OutputID, size int64, perm os.FileMode) error { + name := c.fileName(out, "d") // TODO(matloob): use a different suffix for the executable cache? info, err := os.Stat(name) + if executableName != "" { + // This is an executable file. The file at name won't hold the output itself, but will + // be a directory that holds the output, named according to executableName. Check to see + // if the directory already exists, and if it does not, create it. Then reset name + // to the name we want the output written to. + if err != nil { + if !os.IsNotExist(err) { + return err + } + if err := os.Mkdir(name, 0o777); err != nil { + return err + } + if info, err = os.Stat(name); err != nil { + return err + } + } + if !info.IsDir() { + return errors.New("internal error: invalid binary cache entry: not a directory") + } + + // directory exists. now set name to the inner file + name = filepath.Join(name, executableName) + info, err = os.Stat(name) + } if err == nil && info.Size() == size { // Check hash. if f, err := os.Open(name); err == nil { @@ -585,8 +631,14 @@ func (c *DiskCache) copyFile(file io.ReadSeeker, out OutputID, size int64) error if err == nil && info.Size() > size { // shouldn't happen but fix in case mode |= os.O_TRUNC } - f, err := os.OpenFile(name, mode, 0666) + f, err := os.OpenFile(name, mode, perm) if err != nil { + if base.IsETXTBSY(err) { + // This file is being used by an executable. It must have + // already been written by another go process and then run. + // return without an error. + return nil + } return err } defer f.Close() diff --git a/internal/go/cache/cache_test.go b/internal/go/cache/cache_test.go index 185dafc3aaf6..e1752ff7e287 100644 --- a/internal/go/cache/cache_test.go +++ b/internal/go/cache/cache_test.go @@ -21,18 +21,14 @@ func init() { } func TestBasic(t *testing.T) { - dir, err := os.MkdirTemp("", "cachetest-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - _, err = Open(filepath.Join(dir, "notexist")) + dir := t.TempDir() + _, err := Open(filepath.Join(dir, "notexist")) if err == nil { t.Fatal(`Open("tmp/notexist") succeeded, want failure`) } cdir := filepath.Join(dir, "c1") - if err := os.Mkdir(cdir, 0744); err != nil { + if err := os.Mkdir(cdir, 0777); err != nil { t.Fatal(err) } @@ -66,13 +62,7 @@ func TestBasic(t *testing.T) { } func TestGrowth(t *testing.T) { - dir, err := os.MkdirTemp("", "cachetest-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - c, err := Open(dir) + c, err := Open(t.TempDir()) if err != nil { t.Fatalf("Open: %v", err) } @@ -119,13 +109,7 @@ func TestGrowth(t *testing.T) { // t.Fatal("initEnv did not set verify") // } // -// dir, err := os.MkdirTemp("", "cachetest-") -// if err != nil { -// t.Fatal(err) -// } -// defer os.RemoveAll(dir) -// -// c, err := Open(dir) +// c, err := Open(t.TempDir()) // if err != nil { // t.Fatalf("Open: %v", err) // } @@ -152,12 +136,7 @@ func dummyID(x int) [HashSize]byte { } func TestCacheTrim(t *testing.T) { - dir, err := os.MkdirTemp("", "cachetest-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - + dir := t.TempDir() c, err := Open(dir) if err != nil { t.Fatalf("Open: %v", err) diff --git a/internal/go/cache/default.go b/internal/go/cache/default.go index 7232f1ef3e60..cf38ab3d7b9e 100644 --- a/internal/go/cache/default.go +++ b/internal/go/cache/default.go @@ -15,14 +15,10 @@ import ( // Default returns the default cache to use. // It never returns nil. func Default() Cache { - defaultOnce.Do(initDefaultCache) - return defaultCache + return initDefaultCacheOnce() } -var ( - defaultOnce sync.Once - defaultCache Cache -) +var initDefaultCacheOnce = sync.OnceValue(initDefaultCache) // cacheREADME is a message stored in a README in the cache directory. // Because the cache lives outside the normal Go trees, we leave the @@ -32,22 +28,20 @@ const cacheREADME = `This directory holds cached build artifacts from golangci-l // initDefaultCache does the work of finding the default cache // the first time Default is called. -func initDefaultCache() { +func initDefaultCache() Cache { dir, _ := DefaultDir() if dir == "off" { if defaultDirErr != nil { base.Fatalf("build cache is required, but could not be located: %v", defaultDirErr) } - base.Fatalf("build cache is disabled by %s=off, but required", envGolangciLintCache) + base.Fatalf("build cache is disabled by %s=off, but required as of Go 1.12", envGolangciLintCache) } - if err := os.MkdirAll(dir, 0744); err != nil { + if err := os.MkdirAll(dir, 0o777); err != nil { base.Fatalf("failed to initialize build cache at %s: %s\n", dir, err) } if _, err := os.Stat(filepath.Join(dir, "README")); err != nil { // Best effort. - if wErr := os.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666); wErr != nil { - base.Fatalf("Failed to write README file to cache dir %s: %s", dir, err) - } + os.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666) } diskCache, err := Open(dir) @@ -56,10 +50,10 @@ func initDefaultCache() { } if v := os.Getenv(envGolangciLintCacheProg); v != "" { - defaultCache = startCacheProg(v, diskCache) - } else { - defaultCache = diskCache + return startCacheProg(v, diskCache) } + + return diskCache } var ( @@ -74,7 +68,7 @@ var ( // and reports whether the effective value differs from GOLANGCI_LINT_CACHE. func DefaultDir() (string, bool) { // Save the result of the first call to DefaultDir for later use in - // initDefaultCache. cmd/go/main.go explicitly sets GOCACHE so that + // initDefaultCache. cmd/go/main.go explicitly sets GOLANGCI_LINT_CACHE so that // subprocesses will inherit it, but that means initDefaultCache can't // otherwise distinguish between an explicit "off" and a UserCacheDir error. diff --git a/internal/go/cache/hash.go b/internal/go/cache/hash.go index d5169dd4c491..6a53dd886757 100644 --- a/internal/go/cache/hash.go +++ b/internal/go/cache/hash.go @@ -50,7 +50,7 @@ func stripExperiment(version string) string { // action ID with a string description of the subkey. func Subkey(parent ActionID, desc string) (ActionID, error) { h := sha256.New() - h.Write([]byte(("subkey:"))) + h.Write([]byte("subkey:")) n, err := h.Write(parent[:]) if n != len(parent) { return ActionID{}, fmt.Errorf("wrote %d/%d bytes of parent with error %s", n, len(parent), err) diff --git a/internal/go/cache/prog.go b/internal/go/cache/prog.go index 8b317b021073..dc44e1385b9d 100644 --- a/internal/go/cache/prog.go +++ b/internal/go/cache/prog.go @@ -21,6 +21,7 @@ import ( "sync/atomic" "time" + "github.com/golangci/golangci-lint/v2/internal/go/cacheprog" "github.com/golangci/golangci-lint/v2/internal/go/quoted" ) @@ -38,7 +39,7 @@ type ProgCache struct { // can are the commands that the child process declared that it supports. // This is effectively the versioning mechanism. - can map[ProgCmd]bool + can map[cacheprog.Cmd]bool // fuzzDirCache is another Cache implementation to use for the FuzzDir // method. In practice this is the default GOCACHE disk-based @@ -55,7 +56,7 @@ type ProgCache struct { mu sync.Mutex // guards following fields nextID int64 - inFlight map[int64]chan<- *ProgResponse + inFlight map[int64]chan<- *cacheprog.Response outputFile map[OutputID]string // object => abs path on disk // writeMu serializes writing to the child process. @@ -63,84 +64,6 @@ type ProgCache struct { writeMu sync.Mutex } -// ProgCmd is a command that can be issued to a child process. -// -// If the interface needs to grow, we can add new commands or new versioned -// commands like "get2". -type ProgCmd string - -const ( - cmdGet = ProgCmd("get") - cmdPut = ProgCmd("put") - cmdClose = ProgCmd("close") -) - -// ProgRequest is the JSON-encoded message that's sent from cmd/go to -// the GOLANGCI_LINT_CACHEPROG child process over stdin. Each JSON object is on its -// own line. A ProgRequest of Type "put" with BodySize > 0 will be followed -// by a line containing a base64-encoded JSON string literal of the body. -type ProgRequest struct { - // ID is a unique number per process across all requests. - // It must be echoed in the ProgResponse from the child. - ID int64 - - // Command is the type of request. - // The cmd/go tool will only send commands that were declared - // as supported by the child. - Command ProgCmd - - // ActionID is non-nil for get and puts. - ActionID []byte `json:",omitempty"` // or nil if not used - - // ObjectID is set for Type "put" and "output-file". - ObjectID []byte `json:",omitempty"` // or nil if not used - - // Body is the body for "put" requests. It's sent after the JSON object - // as a base64-encoded JSON string when BodySize is non-zero. - // It's sent as a separate JSON value instead of being a struct field - // send in this JSON object so large values can be streamed in both directions. - // The base64 string body of a ProgRequest will always be written - // immediately after the JSON object and a newline. - Body io.Reader `json:"-"` - - // BodySize is the number of bytes of Body. If zero, the body isn't written. - BodySize int64 `json:",omitempty"` -} - -// ProgResponse is the JSON response from the child process to cmd/go. -// -// With the exception of the first protocol message that the child writes to its -// stdout with ID==0 and KnownCommands populated, these are only sent in -// response to a ProgRequest from cmd/go. -// -// ProgResponses can be sent in any order. The ID must match the request they're -// replying to. -type ProgResponse struct { - ID int64 // that corresponds to ProgRequest; they can be answered out of order - Err string `json:",omitempty"` // if non-empty, the error - - // KnownCommands is included in the first message that cache helper program - // writes to stdout on startup (with ID==0). It includes the - // ProgRequest.Command types that are supported by the program. - // - // This lets us extend the protocol gracefully over time (adding "get2", - // etc), or fail gracefully when needed. It also lets us verify the program - // wants to be a cache helper. - KnownCommands []ProgCmd `json:",omitempty"` - - // For Get requests. - - Miss bool `json:",omitempty"` // cache miss - OutputID []byte `json:",omitempty"` - Size int64 `json:",omitempty"` // in bytes - Time *time.Time `json:",omitempty"` // an Entry.Time; when the object was added to the docs - - // DiskPath is the absolute path on disk of the ObjectID corresponding - // a "get" request's ActionID (on cache hit) or a "put" request's - // provided ObjectID. - DiskPath string `json:",omitempty"` -} - // startCacheProg starts the prog binary (with optional space-separated flags) // and returns a Cache implementation that talks to it. // @@ -165,13 +88,15 @@ func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache { cmd := exec.CommandContext(ctx, prog, args...) out, err := cmd.StdoutPipe() if err != nil { - base.Fatalf("StdoutPipe to %s: %v", envGolangciLintCacheProg, err) + base.Fatalf("StdoutPipe to %s: envGolangciLintCacheProg, %v", envGolangciLintCacheProg, err) } in, err := cmd.StdinPipe() if err != nil { - base.Fatalf("StdinPipe to %s: %v", envGolangciLintCacheProg, err) + base.Fatalf("StdinPipe to %s: envGolangciLintCacheProg, %v", envGolangciLintCacheProg, err) } cmd.Stderr = os.Stderr + // On close, we cancel the context. Rather than killing the helper, + // close its stdin. cmd.Cancel = in.Close if err := cmd.Start(); err != nil { @@ -186,14 +111,14 @@ func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache { stdout: out, stdin: in, bw: bufio.NewWriter(in), - inFlight: make(map[int64]chan<- *ProgResponse), + inFlight: make(map[int64]chan<- *cacheprog.Response), outputFile: make(map[OutputID]string), readLoopDone: make(chan struct{}), } // Register our interest in the initial protocol message from the child to // us, saying what it can do. - capResc := make(chan *ProgResponse, 1) + capResc := make(chan *cacheprog.Response, 1) pc.inFlight[0] = capResc pc.jenc = json.NewEncoder(pc.bw) @@ -208,7 +133,7 @@ func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache { case <-timer.C: log.Printf("# still waiting for %s %v ...", envGolangciLintCacheProg, prog) case capRes := <-capResc: - can := map[ProgCmd]bool{} + can := map[cacheprog.Cmd]bool{} for _, cmd := range capRes.KnownCommands { can[cmd] = true } @@ -225,9 +150,15 @@ func (c *ProgCache) readLoop(readLoopDone chan<- struct{}) { defer close(readLoopDone) jd := json.NewDecoder(c.stdout) for { - res := new(ProgResponse) + res := new(cacheprog.Response) if err := jd.Decode(res); err != nil { if c.closing.Load() { + c.mu.Lock() + for _, ch := range c.inFlight { + close(ch) + } + c.inFlight = nil + c.mu.Unlock() return // quietly } if err == io.EOF { @@ -250,13 +181,18 @@ func (c *ProgCache) readLoop(readLoopDone chan<- struct{}) { } } -func (c *ProgCache) send(ctx context.Context, req *ProgRequest) (*ProgResponse, error) { - resc := make(chan *ProgResponse, 1) +var errCacheprogClosed = fmt.Errorf("%s program closed unexpectedly", envGolangciLintCacheProg) + +func (c *ProgCache) send(ctx context.Context, req *cacheprog.Request) (*cacheprog.Response, error) { + resc := make(chan *cacheprog.Response, 1) if err := c.writeToChild(req, resc); err != nil { return nil, err } select { case res := <-resc: + if res == nil { + return nil, errCacheprogClosed + } if res.Err != "" { return nil, errors.New(res.Err) } @@ -266,8 +202,11 @@ func (c *ProgCache) send(ctx context.Context, req *ProgRequest) (*ProgResponse, } } -func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (err error) { +func (c *ProgCache) writeToChild(req *cacheprog.Request, resc chan<- *cacheprog.Response) (err error) { c.mu.Lock() + if c.inFlight == nil { + return errCacheprogClosed + } c.nextID++ req.ID = c.nextID c.inFlight[req.ID] = resc @@ -276,7 +215,9 @@ func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (e defer func() { if err != nil { c.mu.Lock() - delete(c.inFlight, req.ID) + if c.inFlight != nil { + delete(c.inFlight, req.ID) + } c.mu.Unlock() } }() @@ -303,8 +244,8 @@ func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (e return nil } if wrote != req.BodySize { - return fmt.Errorf("short write writing body to %s for action %x, object %x: wrote %v; expected %v", - envGolangciLintCacheProg, req.ActionID, req.ObjectID, wrote, req.BodySize) + return fmt.Errorf("short write writing body to %s for action %x, output %x: wrote %v; expected %v", + envGolangciLintCacheProg, req.ActionID, req.OutputID, wrote, req.BodySize) } if _, err := c.bw.WriteString("\"\n"); err != nil { return err @@ -317,7 +258,7 @@ func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (e } func (c *ProgCache) Get(a ActionID) (Entry, error) { - if !c.can[cmdGet] { + if !c.can[cacheprog.CmdGet] { // They can't do a "get". Maybe they're a write-only cache. // // TODO(bradfitz,bcmills): figure out the proper error type here. Maybe @@ -327,8 +268,8 @@ func (c *ProgCache) Get(a ActionID) (Entry, error) { // error types on the Cache interface. return Entry{}, &entryNotFoundError{} } - res, err := c.send(c.ctx, &ProgRequest{ - Command: cmdGet, + res, err := c.send(c.ctx, &cacheprog.Request{ + Command: cacheprog.CmdGet, ActionID: a[:], }) if err != nil { @@ -384,15 +325,15 @@ func (c *ProgCache) Put(a ActionID, file io.ReadSeeker) (_ OutputID, size int64, return OutputID{}, 0, err } - if !c.can[cmdPut] { + if !c.can[cacheprog.CmdPut] { // Child is a read-only cache. Do nothing. return out, size, nil } - res, err := c.send(c.ctx, &ProgRequest{ - Command: cmdPut, + res, err := c.send(c.ctx, &cacheprog.Request{ + Command: cacheprog.CmdPut, ActionID: a[:], - ObjectID: out[:], + OutputID: out[:], Body: file, BodySize: size, }) @@ -413,10 +354,16 @@ func (c *ProgCache) Close() error { // First write a "close" message to the child so it can exit nicely // and clean up if it wants. Only after that exchange do we cancel // the context that kills the process. - if c.can[cmdClose] { - _, err = c.send(c.ctx, &ProgRequest{Command: cmdClose}) + if c.can[cacheprog.CmdClose] { + _, err = c.send(c.ctx, &cacheprog.Request{Command: cacheprog.CmdClose}) + if errors.Is(err, errCacheprogClosed) { + // Allow the child to quit without responding to close. + err = nil + } } + // Cancel the context, which will close the helper's stdin. c.ctxCancel() + // Wait until the helper closes its stdout. <-c.readLoopDone return err } diff --git a/internal/go/cache/readme.md b/internal/go/cache/readme.md index 5be600e42525..66341fd4b8df 100644 --- a/internal/go/cache/readme.md +++ b/internal/go/cache/readme.md @@ -12,6 +12,8 @@ The main modifications are: ## History +- https://github.com/golangci/golangci-lint/pull/5576 + - sync go1.24.1 - https://github.com/golangci/golangci-lint/pull/5100 - Move package from `internal/cache` to `internal/go/cache` - https://github.com/golangci/golangci-lint/pull/5098 diff --git a/internal/go/cacheprog/cacheprog.go b/internal/go/cacheprog/cacheprog.go new file mode 100644 index 000000000000..a2796592df5f --- /dev/null +++ b/internal/go/cacheprog/cacheprog.go @@ -0,0 +1,137 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cacheprog defines the protocol for a GOCACHEPROG program. +// +// By default, the go command manages a build cache stored in the file system +// itself. GOCACHEPROG can be set to the name of a command (with optional +// space-separated flags) that implements the go command build cache externally. +// This permits defining a different cache policy. +// +// The go command will start the GOCACHEPROG as a subprocess and communicate +// with it via JSON messages over stdin/stdout. The subprocess's stderr will be +// connected to the go command's stderr. +// +// The subprocess should immediately send a [Response] with its capabilities. +// After that, the go command will send a stream of [Request] messages and the +// subprocess should reply to each [Request] with a [Response] message. +package cacheprog + +import ( + "io" + "time" +) + +// Cmd is a command that can be issued to a child process. +// +// If the interface needs to grow, the go command can add new commands or new +// versioned commands like "get2" in the future. The initial [Response] from +// the child process indicates which commands it supports. +type Cmd string + +const ( + // CmdPut tells the cache program to store an object in the cache. + // + // [Request.ActionID] is the cache key of this object. The cache should + // store [Request.OutputID] and [Request.Body] under this key for a + // later "get" request. It must also store the Body in a file in the local + // file system and return the path to that file in [Response.DiskPath], + // which must exist at least until a "close" request. + CmdPut = Cmd("put") + + // CmdGet tells the cache program to retrieve an object from the cache. + // + // [Request.ActionID] specifies the key of the object to get. If the + // cache does not contain this object, it should set [Response.Miss] to + // true. Otherwise, it should populate the fields of [Response], + // including setting [Response.OutputID] to the OutputID of the original + // "put" request and [Response.DiskPath] to the path of a local file + // containing the Body of the original "put" request. That file must + // continue to exist at least until a "close" request. + CmdGet = Cmd("get") + + // CmdClose requests that the cache program exit gracefully. + // + // The cache program should reply to this request and then exit + // (thus closing its stdout). + CmdClose = Cmd("close") +) + +// Request is the JSON-encoded message that's sent from the go command to +// the GOCACHEPROG child process over stdin. Each JSON object is on its own +// line. A ProgRequest of Type "put" with BodySize > 0 will be followed by a +// line containing a base64-encoded JSON string literal of the body. +type Request struct { + // ID is a unique number per process across all requests. + // It must be echoed in the Response from the child. + ID int64 + + // Command is the type of request. + // The go command will only send commands that were declared + // as supported by the child. + Command Cmd + + // ActionID is the cache key for "put" and "get" requests. + ActionID []byte `json:",omitempty"` // or nil if not used + + // OutputID is stored with the body for "put" requests. + // + // Prior to Go 1.24, when GOCACHEPROG was still an experiment, this was + // accidentally named ObjectID. It was renamed to OutputID in Go 1.24. + OutputID []byte `json:",omitempty"` // or nil if not used + + // Body is the body for "put" requests. It's sent after the JSON object + // as a base64-encoded JSON string when BodySize is non-zero. + // It's sent as a separate JSON value instead of being a struct field + // send in this JSON object so large values can be streamed in both directions. + // The base64 string body of a Request will always be written + // immediately after the JSON object and a newline. + Body io.Reader `json:"-"` + + // BodySize is the number of bytes of Body. If zero, the body isn't written. + BodySize int64 `json:",omitempty"` + + // ObjectID is the accidental spelling of OutputID that was used prior to Go + // 1.24. + // + // Deprecated: use OutputID. This field is only populated temporarily for + // backwards compatibility with Go 1.23 and earlier when + // GOEXPERIMENT=gocacheprog is set. It will be removed in Go 1.25. + ObjectID []byte `json:",omitempty"` +} + +// Response is the JSON response from the child process to the go command. +// +// With the exception of the first protocol message that the child writes to its +// stdout with ID==0 and KnownCommands populated, these are only sent in +// response to a Request from the go command. +// +// Responses can be sent in any order. The ID must match the request they're +// replying to. +type Response struct { + ID int64 // that corresponds to Request; they can be answered out of order + Err string `json:",omitempty"` // if non-empty, the error + + // KnownCommands is included in the first message that cache helper program + // writes to stdout on startup (with ID==0). It includes the + // Request.Command types that are supported by the program. + // + // This lets the go command extend the protocol gracefully over time (adding + // "get2", etc), or fail gracefully when needed. It also lets the go command + // verify the program wants to be a cache helper. + KnownCommands []Cmd `json:",omitempty"` + + // For "get" requests. + + Miss bool `json:",omitempty"` // cache miss + OutputID []byte `json:",omitempty"` // the ObjectID stored with the body + Size int64 `json:",omitempty"` // body size in bytes + Time *time.Time `json:",omitempty"` // when the object was put in the cache (optional; used for cache expiration) + + // For "get" and "put" requests. + + // DiskPath is the absolute path on disk of the body corresponding to a + // "get" (on cache hit) or "put" request's ActionID. + DiskPath string `json:",omitempty"` +} diff --git a/internal/go/cacheprog/readme.md b/internal/go/cacheprog/readme.md new file mode 100644 index 000000000000..1b08c8480620 --- /dev/null +++ b/internal/go/cacheprog/readme.md @@ -0,0 +1,9 @@ +# quoted + +Extracted from `go/src/cmd/go/internal/cacheprog/` (related to `cache`). +This is just a copy of the Go code without any changes. + +## History + +- https://github.com/golangci/golangci-lint/pull/5576 + - sync go1.24.1 diff --git a/internal/go/mmap/mmap.go b/internal/go/mmap/mmap.go index fcbd3e08c1c5..fd374df82efa 100644 --- a/internal/go/mmap/mmap.go +++ b/internal/go/mmap/mmap.go @@ -22,10 +22,11 @@ type Data struct { } // Mmap maps the given file into memory. -func Mmap(file string) (Data, error) { +func Mmap(file string) (Data, bool, error) { f, err := os.Open(file) if err != nil { - return Data{}, err + return Data{}, false, err } - return mmapFile(f) + data, err := mmapFile(f) + return data, true, err } diff --git a/internal/go/mmap/mmap_windows.go b/internal/go/mmap/mmap_windows.go index 479ee307544f..256fab4b4949 100644 --- a/internal/go/mmap/mmap_windows.go +++ b/internal/go/mmap/mmap_windows.go @@ -37,5 +37,11 @@ func mmapFile(f *os.File) (Data, error) { return Data{}, fmt.Errorf("VirtualQuery %s: %w", f.Name(), err) } data := unsafe.Slice((*byte)(unsafe.Pointer(addr)), int(info.RegionSize)) - return Data{f, data}, nil + if len(data) < int(size) { + // In some cases, especially on 386, we may not receive a in incomplete mapping: + // one that is shorter than the file itself. Return an error in those cases because + // incomplete mappings are not useful. + return Data{}, fmt.Errorf("mmapFile: received incomplete mapping of file") + } + return Data{f, data[:int(size)]}, nil } diff --git a/internal/go/mmap/readme.md b/internal/go/mmap/readme.md index f68aef097c88..5cbfdeefe677 100644 --- a/internal/go/mmap/readme.md +++ b/internal/go/mmap/readme.md @@ -5,6 +5,8 @@ This is just a copy of the Go code without any changes. ## History +- https://github.com/golangci/golangci-lint/pull/5576 + - sync go1.24.1 - https://github.com/golangci/golangci-lint/pull/5100 - Move package from `internal/mmap` to `internal/go/mmap` - https://github.com/golangci/golangci-lint/pull/5098 diff --git a/internal/go/quoted/readme.md b/internal/go/quoted/readme.md index a5e4c4bb3b09..97868185c030 100644 --- a/internal/go/quoted/readme.md +++ b/internal/go/quoted/readme.md @@ -5,6 +5,8 @@ This is just a copy of the Go code without any changes. ## History +- https://github.com/golangci/golangci-lint/pull/5576 + - sync go1.24.1 (no change) - https://github.com/golangci/golangci-lint/pull/5100 - Move package from `internal/quoted` to `internal/go/quoted` - https://github.com/golangci/golangci-lint/pull/5098 diff --git a/internal/go/robustio/readme.md b/internal/go/robustio/readme.md deleted file mode 100644 index f4dbc162649d..000000000000 --- a/internal/go/robustio/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -# robustio - -Extracted from go1.19.1/src/cmd/go/internal/robustio - -There is only one modification: -- ERROR_SHARING_VIOLATION extracted from go1.19.1/src/internal/syscall/windows/syscall_windows.go to remove the dependencies to `internal/syscall/windows` - -## History - -- https://github.com/golangci/golangci-lint/pull/5100 - - Move package from `internal/robustio` to `internal/go/robustio` diff --git a/internal/go/robustio/robustio.go b/internal/go/robustio/robustio.go deleted file mode 100644 index 15b33773cf5f..000000000000 --- a/internal/go/robustio/robustio.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package robustio wraps I/O functions that are prone to failure on Windows, -// transparently retrying errors up to an arbitrary timeout. -// -// Errors are classified heuristically and retries are bounded, so the functions -// in this package do not completely eliminate spurious errors. However, they do -// significantly reduce the rate of failure in practice. -// -// If so, the error will likely wrap one of: -// The functions in this package do not completely eliminate spurious errors, -// but substantially reduce their rate of occurrence in practice. -package robustio - -// Rename is like os.Rename, but on Windows retries errors that may occur if the -// file is concurrently read or overwritten. -// -// (See golang.org/issue/31247 and golang.org/issue/32188.) -func Rename(oldpath, newpath string) error { - return rename(oldpath, newpath) -} - -// ReadFile is like os.ReadFile, but on Windows retries errors that may -// occur if the file is concurrently replaced. -// -// (See golang.org/issue/31247 and golang.org/issue/32188.) -func ReadFile(filename string) ([]byte, error) { - return readFile(filename) -} - -// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur -// if an executable file in the directory has recently been executed. -// -// (See golang.org/issue/19491.) -func RemoveAll(path string) error { - return removeAll(path) -} - -// IsEphemeralError reports whether err is one of the errors that the functions -// in this package attempt to mitigate. -// -// Errors considered ephemeral include: -// - syscall.ERROR_ACCESS_DENIED -// - syscall.ERROR_FILE_NOT_FOUND -// - internal/syscall/windows.ERROR_SHARING_VIOLATION -// -// This set may be expanded in the future; programs must not rely on the -// non-ephemerality of any given error. -func IsEphemeralError(err error) bool { - return isEphemeralError(err) -} diff --git a/internal/go/robustio/robustio_darwin.go b/internal/go/robustio/robustio_darwin.go deleted file mode 100644 index 99fd8ebc2fff..000000000000 --- a/internal/go/robustio/robustio_darwin.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package robustio - -import ( - "errors" - "syscall" -) - -const errFileNotFound = syscall.ENOENT - -// isEphemeralError returns true if err may be resolved by waiting. -func isEphemeralError(err error) bool { - var errno syscall.Errno - if errors.As(err, &errno) { - return errno == errFileNotFound - } - return false -} diff --git a/internal/go/robustio/robustio_flaky.go b/internal/go/robustio/robustio_flaky.go deleted file mode 100644 index c56e36ca6241..000000000000 --- a/internal/go/robustio/robustio_flaky.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build windows || darwin - -package robustio - -import ( - "errors" - "math/rand" - "os" - "syscall" - "time" -) - -const arbitraryTimeout = 2000 * time.Millisecond - -// retry retries ephemeral errors from f up to an arbitrary timeout -// to work around filesystem flakiness on Windows and Darwin. -func retry(f func() (err error, mayRetry bool)) error { - var ( - bestErr error - lowestErrno syscall.Errno - start time.Time - nextSleep time.Duration = 1 * time.Millisecond - ) - for { - err, mayRetry := f() - if err == nil || !mayRetry { - return err - } - - var errno syscall.Errno - if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) { - bestErr = err - lowestErrno = errno - } else if bestErr == nil { - bestErr = err - } - - if start.IsZero() { - start = time.Now() - } else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout { - break - } - time.Sleep(nextSleep) - nextSleep += time.Duration(rand.Int63n(int64(nextSleep))) - } - - return bestErr -} - -// rename is like os.Rename, but retries ephemeral errors. -// -// On Windows it wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with -// MOVEFILE_REPLACE_EXISTING. -// -// Windows also provides a different system call, ReplaceFile, -// that provides similar semantics, but perhaps preserves more metadata. (The -// documentation on the differences between the two is very sparse.) -// -// Empirical error rates with MoveFileEx are lower under modest concurrency, so -// for now we're sticking with what the os package already provides. -func rename(oldpath, newpath string) (err error) { - return retry(func() (err error, mayRetry bool) { - err = os.Rename(oldpath, newpath) - return err, isEphemeralError(err) - }) -} - -// readFile is like os.ReadFile, but retries ephemeral errors. -func readFile(filename string) ([]byte, error) { - var b []byte - err := retry(func() (err error, mayRetry bool) { - b, err = os.ReadFile(filename) - - // Unlike in rename, we do not retry errFileNotFound here: it can occur - // as a spurious error, but the file may also genuinely not exist, so the - // increase in robustness is probably not worth the extra latency. - return err, isEphemeralError(err) && !errors.Is(err, errFileNotFound) - }) - return b, err -} - -func removeAll(path string) error { - return retry(func() (err error, mayRetry bool) { - err = os.RemoveAll(path) - return err, isEphemeralError(err) - }) -} diff --git a/internal/go/robustio/robustio_other.go b/internal/go/robustio/robustio_other.go deleted file mode 100644 index da9a46e4face..000000000000 --- a/internal/go/robustio/robustio_other.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !windows && !darwin - -package robustio - -import ( - "os" -) - -func rename(oldpath, newpath string) error { - return os.Rename(oldpath, newpath) -} - -func readFile(filename string) ([]byte, error) { - return os.ReadFile(filename) -} - -func removeAll(path string) error { - return os.RemoveAll(path) -} - -func isEphemeralError(err error) bool { - return false -} diff --git a/internal/go/robustio/robustio_windows.go b/internal/go/robustio/robustio_windows.go deleted file mode 100644 index fe1728954c27..000000000000 --- a/internal/go/robustio/robustio_windows.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package robustio - -import ( - "errors" - "syscall" -) - -const errFileNotFound = syscall.ERROR_FILE_NOT_FOUND - -// ERROR_SHARING_VIOLATION (ldez) extract from go1.19.1/src/internal/syscall/windows/syscall_windows.go. -// This is the only modification of this file. -const ERROR_SHARING_VIOLATION syscall.Errno = 32 - -// isEphemeralError returns true if err may be resolved by waiting. -func isEphemeralError(err error) bool { - var errno syscall.Errno - if errors.As(err, &errno) { - switch errno { - case syscall.ERROR_ACCESS_DENIED, - syscall.ERROR_FILE_NOT_FOUND, - ERROR_SHARING_VIOLATION: - return true - } - } - return false -} diff --git a/internal/go/testenv/readme.md b/internal/go/testenv/readme.md index 6610210ee193..dc4f5d3bee1f 100644 --- a/internal/go/testenv/readme.md +++ b/internal/go/testenv/readme.md @@ -6,6 +6,8 @@ Only the function `SyscallIsNotSupported` is extracted (related to `cache`). ## History +- https://github.com/golangci/golangci-lint/pull/5576 + - sync go1.24.1 - https://github.com/golangci/golangci-lint/pull/5100 - Move package from `internal/testenv` to `internal/go/testenv` - https://github.com/golangci/golangci-lint/pull/5098 diff --git a/internal/go/testenv/testenv_notwin.go b/internal/go/testenv/testenv_notwin.go index 30e159a6ecd4..9dddea94d057 100644 --- a/internal/go/testenv/testenv_notwin.go +++ b/internal/go/testenv/testenv_notwin.go @@ -11,9 +11,10 @@ import ( "os" "path/filepath" "runtime" + "sync" ) -func hasSymlink() (ok bool, reason string) { +var hasSymlink = sync.OnceValues(func() (ok bool, reason string) { switch runtime.GOOS { case "plan9": return false, "" @@ -43,4 +44,4 @@ func hasSymlink() (ok bool, reason string) { } return true, "" -} +}) diff --git a/internal/go/testenv/testenv_windows.go b/internal/go/testenv/testenv_windows.go index 4802b139518e..eed53cdfb2b9 100644 --- a/internal/go/testenv/testenv_windows.go +++ b/internal/go/testenv/testenv_windows.go @@ -5,16 +5,14 @@ package testenv import ( + "errors" "os" "path/filepath" "sync" "syscall" ) -var symlinkOnce sync.Once -var winSymlinkErr error - -func initWinHasSymlink() { +var hasSymlink = sync.OnceValues(func() (bool, string) { tmpdir, err := os.MkdirTemp("", "symtest") if err != nil { panic("failed to create temp directory: " + err.Error()) @@ -22,26 +20,13 @@ func initWinHasSymlink() { defer os.RemoveAll(tmpdir) err = os.Symlink("target", filepath.Join(tmpdir, "symlink")) - if err != nil { - err = err.(*os.LinkError).Err - switch err { - case syscall.EWINDOWS, syscall.ERROR_PRIVILEGE_NOT_HELD: - winSymlinkErr = err - } - } -} - -func hasSymlink() (ok bool, reason string) { - symlinkOnce.Do(initWinHasSymlink) - - switch winSymlinkErr { - case nil: + switch { + case err == nil: return true, "" - case syscall.EWINDOWS: + case errors.Is(err, syscall.EWINDOWS): return false, ": symlinks are not supported on your version of Windows" - case syscall.ERROR_PRIVILEGE_NOT_HELD: + case errors.Is(err, syscall.ERROR_PRIVILEGE_NOT_HELD): return false, ": you don't have enough privileges to create symlinks" } - return false, "" -} +})