Bugfix: increase max size of embedded config to 100Mb (#4690)

Also rework error handling for better error reporting when the file is
corrupt.

Author: scudette

bin/main.go

@@ -26,8 +26,9 @@ import (
 	"runtime/trace"
 	"time"
 
+	"errors"
+
 	kingpin "github.com/alecthomas/kingpin/v2"
-	errors "github.com/go-errors/errors"
 	"www.velocidex.com/golang/velociraptor/config"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/constants"
@@ -152,6 +153,10 @@ func main() {
 			args = append(args, post...)
 			Prelog("Autoexec with parameters: %v", args)
 		}
+
+		if errors.Is(err, utils.EmbeddedConfigError) {
+			kingpin.FatalIfError(err, "EmbeddedConfigError")
+		}
 	}
 
 	// Log the actual argv that will be run.

config/embedded.go

@@ -3,6 +3,8 @@ package config
 import (
 	"bytes"
 	"compress/zlib"
+	"errors"
+	"fmt"
 	"io"
 	"os"
 
@@ -16,30 +18,34 @@ func ExtractEmbeddedConfig(
 
 	fd, err := os.Open(embedded_file)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w: %v", utils.EmbeddedConfigError, err)
 	}
 
 	// Read a lot of the file into memory so we can extract the
-	// configuration. This solution only loads the first 10mb into
+	// configuration. This solution only loads the first 100mb into
 	// memory which should be sufficient for most practical config
 	// files. If there are embedded binaries they will not be read and
 	// will be ignored at this stage (thay can be extracted with the
 	// 'me' accessor).
-	buf, err := utils.ReadAllWithLimit(fd, 10*1024*1024)
-	if err != nil {
-		return nil, err
+	buf, err := utils.ReadAllWithLimit(fd, 100*1024*1024)
+
+	// It is ok to have a short read here.
+	if err != nil && !errors.Is(err, utils.MemoryBufferExceeded) {
+		return nil, fmt.Errorf("%w: %v", utils.EmbeddedConfigError, err)
 	}
 
 	// Find the embedded marker in the buffer.
 	match := embedded_re.FindIndex(buf)
 	if match == nil {
-		return nil, noEmbeddedConfig
+		return nil, fmt.Errorf("%w: Unable to find signature", utils.EmbeddedConfigError)
 	}
 
 	embedded_string := buf[match[0]:]
 	return decode_embedded_config(embedded_string)
 }
 
+// Read the embedded config from this binary - just uses the static
+// allocated string variable.
 func read_embedded_config() (*config_proto.Config, error) {
 	return decode_embedded_config(FileConfigDefaultYaml)
 }
@@ -49,14 +55,15 @@ func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error)
 	idx := bytes.IndexByte(encoded_string, '\n')
 
 	if len(encoded_string) < idx+10 {
-		return nil, noEmbeddedConfig
+		return nil, fmt.Errorf("%w: embedded file too short", utils.EmbeddedConfigError)
 	}
 
 	// If the following line still starts with # then the file is not
 	// repacked - the repacker will replace all further data with the
 	// compressed string.
 	if encoded_string[idx+1] == '#' {
-		return nil, noEmbeddedConfig
+		return nil, fmt.Errorf("%w: embedded file is not packed yet",
+			utils.EmbeddedConfigError)
 	}
 
 	// Decompress the rest of the data - note that zlib will ignore
@@ -65,20 +72,20 @@ func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error)
 	// whole string here.
 	r, err := zlib.NewReader(bytes.NewReader(encoded_string[idx+1:]))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w: %v", utils.EmbeddedConfigError, err)
 	}
 
 	b := &bytes.Buffer{}
 	_, err = io.Copy(b, r)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w: %v", utils.EmbeddedConfigError, err)
 	}
 	r.Close()
 
 	result := &config_proto.Config{}
 	err = yaml.Unmarshal(b.Bytes(), result)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w: %v", utils.EmbeddedConfigError, err)
 	}
 	return result, nil
 }

config/loader.go

@@ -18,8 +18,7 @@ import (
 )
 
 var (
-	noEmbeddedConfig = errors.New(
-		"No embedded config - you can pack one with the `config repack` command")
+	silentError = errors.New("Silent")
 
 	embedded_re = regexp.MustCompile(`#{3}<Begin Embedded Config>\r?\n`)
 
@@ -312,17 +311,23 @@ func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 	self.loaders = append(self.loaders, loaderFunction{
 		name: "WithEmbedded",
 		loader_func: func(self *Loader) (*config_proto.Config, error) {
+			// Try to get the embedded config inside the binary.
 			if embedded_file == "" {
 				result, err := read_embedded_config()
 				if err != nil {
-					return nil, err
+					// not a critical error - the binary does not have
+					// to have an embedded config
+					return nil, silentError
 				}
 
-				self.Log("Loaded embedded config")
+				self.Log("Loaded embedded config from binary")
 
+				// Set the EmbeddedFile for the "me" accessor- we will
+				// get binaries from this file.
 				EmbeddedFile, err = os.Executable()
 				return result, err
 			}
+
 			// Ensure the "me" accessor uses this file for embedded zip.
 			full_path, err := filepath.Abs(embedded_file)
 			if err != nil {
@@ -334,9 +339,11 @@ func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 			result, err := ExtractEmbeddedConfig(full_path)
 			if err == nil {
 				self.Log("Loaded embedded config from %v", full_path)
+				return result, nil
 			}
-			return result, err
 
+			self.Log("Unable to parse embedded config from %v: %v", full_path, err.Error())
+			return result, HardError{err}
 		}})
 	return self
 }
@@ -492,11 +499,13 @@ func (self *Loader) LoadAndValidate() (*config_proto.Config, error) {
 		}
 
 		// Stop on hard errors.
-		_, ok := err.(HardError)
+		he, ok := err.(HardError)
 		if ok {
-			return nil, err
+			return nil, he.Err
+		}
+		if err != silentError {
+			self.Log("%v", err)
 		}
-		self.Log("%v", err)
 	}
 	return nil, errors.New("Unable to load config from any source.")
 }

services/debug/server/everything.go

@@ -0,0 +1,69 @@
+package server
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"net/http"
+
+	"github.com/Velocidex/ordereddict"
+	"www.velocidex.com/golang/velociraptor/logging"
+	"www.velocidex.com/golang/velociraptor/services"
+	"www.velocidex.com/golang/velociraptor/services/debug"
+	"www.velocidex.com/golang/velociraptor/vql/acl_managers"
+	"www.velocidex.com/golang/vfilter"
+)
+
+func (self *debugMux) handleEverything(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/zip")
+	w.Header().Set("Content-Disposition", "attachment; profiles.zip")
+	w.WriteHeader(200)
+
+	zip_writer := zip.NewWriter(w)
+	defer zip_writer.Close()
+
+	builder := services.ScopeBuilder{
+		Config:     self.config_obj,
+		ACLManager: acl_managers.NullACLManager{},
+		Env:        ordereddict.NewDict(),
+	}
+
+	manager, err := services.GetRepositoryManager(self.config_obj)
+	if err != nil {
+		return
+	}
+
+	scope := manager.BuildScope(builder)
+	defer scope.Close()
+
+	seen := make(map[string]bool)
+
+	for _, i := range debug.GetProfileWriters() {
+		_, pres := seen[i.Name]
+		if pres {
+			logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent)
+			logger.Error("Non unique profile name %v", i.Name)
+		}
+		seen[i.Name] = true
+
+		fd, err := zip_writer.CreateHeader(&zip.FileHeader{
+			Name:   i.Name,
+			Method: zip.Deflate,
+		})
+		if err != nil {
+			continue
+		}
+
+		output_chan := make(chan vfilter.Row)
+		go func() {
+			defer close(output_chan)
+
+			i.ProfileWriter(r.Context(), scope, output_chan)
+		}()
+
+		for row := range output_chan {
+			serialized, _ := json.Marshal(row)
+			serialized = append(serialized, '\n')
+			_, _ = fd.Write(serialized)
+		}
+	}
+}

services/debug/server/handlers.go

@@ -136,6 +136,9 @@ func (self *debugMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case "metrics":
 		promhttp.Handler().ServeHTTP(w, r)
 
+	case "everything":
+		self.handleEverything(w, r)
+
 	default:
 		self.HandleIndex(w, r)
 	}

services/debug/server/template.go

@@ -135,6 +135,7 @@ $(document).ready(function() {
   </head>
 <body>
 <h1>Velociraptor Debug Server</h1>
+<a href="%s/debug/everything">Everything</a>
 <div class="category">Internal</div>
 <ul class="categories">
   <li><a href="%s/debug/metrics">Metrics</a></li>
@@ -202,7 +203,7 @@ func (self *debugMux) renderCategory(node *debug.CategoryTreeNode) string {
 func (self *debugMux) HandleIndex(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 
-	_, _ = w.Write([]byte(fmt.Sprintf(indexHeader, self.base, self.base)))
+	_, _ = w.Write([]byte(fmt.Sprintf(indexHeader, self.base, self.base, self.base)))
 	categories := debug.GetProfileTree()
 	_, _ = w.Write([]byte(self.renderCategory(categories)))
 	_, _ = w.Write([]byte(`</body></html>`))

services/spec.go

@@ -36,6 +36,7 @@ func ClientServicesSpec() *config_proto.ServerServicesConfig {
 		Launcher:            true,
 		HttpCommunicator:    true,
 		ClientEventTable:    true,
+		ClientInfo:          true,
 	}
 }
 

utils/copy.go

@@ -17,15 +17,16 @@ var (
 			return &buffer
 		},
 	}
+
+	MemoryBufferExceeded = Wrap(IOError, "Memory buffer exceeded")
 )
 
-func ReadAllWithLimit(
-	fd io.Reader, limit int) ([]byte, error) {
+func ReadAllWithLimit(fd io.Reader, limit int) ([]byte, error) {
 
 	// If we reach the limit signal this as an error!
 	res, err := ioutil.ReadAll(io.LimitReader(fd, int64(limit)))
 	if len(res) >= limit {
-		return nil, Wrap(IOError, "Memory buffer exceeded")
+		return res, MemoryBufferExceeded
 	}
 
 	return res, err

utils/errors.go

@@ -23,6 +23,7 @@ var (
 	CancelledError      = errors.New("Cancelled")
 	SecretsEnforced     = errors.New("Secrets are enforced - you must specify a secret name")
 	PermissionDenied    = errors.New("PermissionDenied")
+	EmbeddedConfigError = errors.New("EmbeddedConfigError")
 )
 
 // This is a custom error type that wraps an inner error but does not