Added write_file() function as an alias to copy()

accessors/overlay/fixtures/TestOverlay.golden

@@ -1,4 +1,5 @@
 {
  "/file1.txt": "Hello",
- "/file2.txt": "Hello Two"
+ "/file2.txt": "Hello Two",
+ "/subdir/file2.txt": "Hello Subdir"
 }

accessors/overlay/overlay_test.go

@@ -54,6 +54,7 @@ func (self *OverlayAccessorTestSuite) makeFile(path string, content string) {
 func (self *OverlayAccessorTestSuite) TestOverlay() {
 	self.makeFile("foo1/file1.txt", "Hello")
 	self.makeFile("foo2/file2.txt", "Hello Two")
+	self.makeFile("foo2/subdir/file2.txt", "Hello Subdir")
 
 	scope := vql_subsystem.MakeScope().
 		AppendVars(ordereddict.NewDict().
@@ -71,21 +72,29 @@ func (self *OverlayAccessorTestSuite) TestOverlay() {
 	accessor, err := accessors.GetAccessor("overlay", scope)
 	assert.NoError(self.T(), err)
 
-	files, err := accessor.ReadDir("/")
-	assert.NoError(self.T(), err)
-
 	golden := ordereddict.NewDict()
 
-	for _, f := range files {
-		fd, err := accessor.OpenWithOSPath(f.OSPath())
+	check_dir := func(file_path string) {
+		files, err := accessor.ReadDir(file_path)
 		assert.NoError(self.T(), err)
 
-		data, err := utils.ReadAllWithLimit(fd, constants.MAX_MEMORY)
-		assert.NoError(self.T(), err)
-		fd.Close()
+		for _, f := range files {
+			if f.IsDir() {
+				continue
+			}
+
+			fd, err := accessor.OpenWithOSPath(f.OSPath())
+			assert.NoError(self.T(), err)
+
+			data, err := utils.ReadAllWithLimit(fd, constants.MAX_MEMORY)
+			assert.NoError(self.T(), err)
+			fd.Close()
 
-		golden.Set(f.OSPath().String(), string(data))
+			golden.Set(f.OSPath().String(), string(data))
+		}
 	}
+	check_dir("/")
+	check_dir("/subdir/")
 
 	goldie.Assert(self.T(), "TestOverlay", json.MustMarshalIndent(golden))
 

bin/artifacts.go

@@ -160,6 +160,8 @@ func doArtifactCollect() error {
 		return fmt.Errorf("Unable to create config: %w", err)
 	}
 
+	config_obj.Services = services.GenericToolServices()
+
 	ctx, top_cancel := install_sig_handler()
 	defer top_cancel()
 

bin/glob.go

@@ -0,0 +1,20 @@
+package main
+
+import "path/filepath"
+
+func expandOneGlob(path string) []string {
+	res, err := filepath.Glob(path)
+	if err != nil {
+		return []string{path}
+	}
+	return res
+}
+
+// Needed for Windows as the shell does not expand globs.
+func expandGlobs(paths []string) (res []string) {
+	for _, p := range paths {
+		res = append(res, expandOneGlob(p)...)
+	}
+
+	return res
+}

bin/reformat.go

@@ -43,7 +43,7 @@ func doReformat() error {
 	logger := logging.GetLogger(config_obj, &logging.ToolComponent)
 
 	var artifact_paths []string
-	for _, artifact_path := range *reformat_args {
+	for _, artifact_path := range expandGlobs(*reformat_args) {
 		abs, err := filepath.Abs(artifact_path)
 		if err != nil {
 			logger.Error("reformat: could not get absolute path for %v", artifact_path)

bin/verify.go

@@ -52,7 +52,7 @@ func doVerify() error {
 
 	var artifact_paths []string
 
-	for _, artifact_path := range *verify_args {
+	for _, artifact_path := range expandGlobs(*verify_args) {
 		abs, err := filepath.Abs(artifact_path)
 		if err != nil {
 			logger.Error("verify: could not get absolute path for %v", artifact_path)

docs/references/vql.yaml

@@ -8288,7 +8288,6 @@
        FROM range(start=0, end=20)
     }, exit='x=>x._value >= my_limit', inherit=true)
     ```
-
   type: Plugin
   args:
   - name: query
@@ -12507,6 +12506,35 @@
   - linux_amd64_cgo
   - windows_386_cgo
   - windows_amd64_cgo
+- name: write_file
+  description: |
+    Writes a string onto a file.
+
+    This VQL function is a convenience wrapper to the copy() function.
+
+  type: Function
+  args:
+  - name: data
+    type: string
+    description: The data to write
+    required: true
+  - name: dest
+    type: string
+    description: The destination file to write.
+    required: true
+  - name: permissions
+    type: string
+    description: Required permissions (e.g. 'x').
+  - name: append
+    type: bool
+    description: If true we append to the target file otherwise truncate it
+  - name: create_directories
+    type: bool
+    description: If true we ensure the destination directories exist
+  metadata:
+    permissions: FILESYSTEM_WRITE,FILESYSTEM_READ
+  platforms:
+  - linux_amd64_cgo
 - name: write_jsonl
   description: Write a query into a JSONL file.
   type: Plugin

services/client_info/client_info.go

@@ -583,7 +583,11 @@ func (self *ClientInfoManager) Set(
 func NewClientInfoManager(
 	ctx context.Context,
 	wg *sync.WaitGroup,
-	config_obj *config_proto.Config) (*ClientInfoManager, error) {
+	config_obj *config_proto.Config) (services.ClientInfoManager, error) {
+
+	if config_obj.Datastore == nil {
+		return &DummyClientInfoManager{}, nil
+	}
 
 	// Calculate a unique id for each service.
 	service := &ClientInfoManager{
@@ -614,8 +618,6 @@ func NewClientInfoManager(
 
 		<-ctx.Done()
 
-		utils.DlvBreak()
-
 		// When we shut down make sure to save the snapshot.
 		subctx, cancel := utils.WithTimeoutCause(
 			context.Background(), 100*time.Second,
@@ -629,6 +631,11 @@ func NewClientInfoManager(
 		}
 	}()
 
+	err = service.Start(ctx, config_obj, wg)
+	if err != nil {
+		return nil, err
+	}
+
 	return service, nil
 }
 

services/client_info/client_info_test.go

@@ -160,7 +160,7 @@ func (self *ClientInfoTestSuite) TestMasterMinion() {
 		self.Ctx, self.Sm.Wg, minion_config)
 	assert.NoError(self.T(), err)
 
-	err = minion_client_info_manager.Start(
+	err = minion_client_info_manager.(*client_info.ClientInfoManager).Start(
 		self.Ctx, minion_config, self.Sm.Wg)
 	assert.NoError(self.T(), err)
 

services/client_info/dummy.go

@@ -0,0 +1,110 @@
+package client_info
+
+import (
+	"context"
+
+	"github.com/Velocidex/ordereddict"
+	crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto"
+	"www.velocidex.com/golang/velociraptor/services"
+	"www.velocidex.com/golang/velociraptor/utils"
+)
+
+type DummyClientInfoManager struct{}
+
+func (self DummyClientInfoManager) ListClients(ctx context.Context) <-chan string {
+	output_chan := make(chan string)
+	close(output_chan)
+	return output_chan
+}
+
+// Used to set a new client record. To modify an existing record -
+// or set a new one use Modify()
+func (self DummyClientInfoManager) Set(ctx context.Context,
+	client_info *services.ClientInfo) error {
+	return utils.NotImplementedError
+}
+
+// Modify a record or set a new one - if the record is not found,
+// modifier will receive a nil client_info. The ClientInfoManager
+// can not be accessed within the modifier function as it is
+// locked for the duration of the change.
+func (self DummyClientInfoManager) Modify(ctx context.Context, client_id string,
+	modifier func(client_info *services.ClientInfo) (
+		new_record *services.ClientInfo, err error)) error {
+	return utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) Get(ctx context.Context,
+	client_id string) (*services.ClientInfo, error) {
+	return nil, utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) Remove(ctx context.Context, client_id string) {}
+
+func (self DummyClientInfoManager) GetStats(
+	ctx context.Context, client_id string) (*services.Stats, error) {
+	return nil, utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) UpdateStats(
+	ctx context.Context, client_id string, stats *services.Stats) error {
+	return utils.NotImplementedError
+}
+
+// Get the client's tasks and remove them from the queue.
+func (self DummyClientInfoManager) GetClientTasks(ctx context.Context,
+	client_id string) ([]*crypto_proto.VeloMessage, error) {
+	return nil, utils.NotImplementedError
+}
+
+// Get all the tasks without de-queuing them.
+func (self DummyClientInfoManager) PeekClientTasks(ctx context.Context,
+	client_id string) ([]*crypto_proto.VeloMessage, error) {
+	return nil, utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) QueueMessagesForClient(
+	ctx context.Context,
+	client_id string,
+	req []*crypto_proto.VeloMessage,
+	notify bool, /* Also notify the client about the new task */
+) error {
+	return utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) QueueMessageForClient(
+	ctx context.Context,
+	client_id string,
+	req *crypto_proto.VeloMessage,
+	notify bool, /* Also notify the client about the new task */
+	completion func()) error {
+	return utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) UnQueueMessageForClient(
+	ctx context.Context,
+	client_id string,
+	req *crypto_proto.VeloMessage) error {
+	return utils.NotImplementedError
+}
+
+// Be able to manipulate the client and server metadata.
+func (self DummyClientInfoManager) GetMetadata(ctx context.Context,
+	client_id string) (*ordereddict.Dict, error) {
+	return nil, utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) SetMetadata(ctx context.Context,
+	client_id string, metadata *ordereddict.Dict, principal string) error {
+	return utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) ValidateClientId(client_id string) error {
+	return utils.NotImplementedError
+}
+
+func (self DummyClientInfoManager) DeleteClient(
+	ctx context.Context, client_id, principal string,
+	progress chan services.DeleteFlowResponse, really_do_it bool) error {
+	return utils.NotImplementedError
+}

services/launcher.go

@@ -84,13 +84,6 @@ func GetLauncher(config_obj *config_proto.Config) (Launcher, error) {
 	}
 
 	svc := org_manager.Services(config_obj.OrgId)
-
-	// We need the client info manager to be up first
-	_, err = svc.ClientInfoManager()
-	if err != nil {
-		return nil, err
-	}
-
 	return svc.Launcher()
 }
 

services/launcher/launcher.go

@@ -826,8 +826,12 @@ func NewLauncherService(
 		return &Launcher{Storage_: &DummyStorer{}}, nil
 	}
 
-	res := &Launcher{
-		Storage_: NewFlowStorageManager(ctx, config_obj, wg),
+	storage, err := NewFlowStorageManager(ctx, config_obj, wg)
+	if err != nil {
+		return nil, err
 	}
-	return res, nil
+
+	return &Launcher{
+		Storage_: storage,
+	}, nil
 }

services/launcher/storage.go

@@ -380,7 +380,7 @@ func (self *FlowStorageManager) GetFlowRequests(
 func NewFlowStorageManager(
 	ctx context.Context,
 	config_obj *config_proto.Config,
-	wg *sync.WaitGroup) *FlowStorageManager {
+	wg *sync.WaitGroup) (*FlowStorageManager, error) {
 	res := &FlowStorageManager{
 		indexBuilders: make(map[string]*flowIndexBuilder),
 		throttler:     utils.NewThrottlerWithDuration(time.Second),
@@ -392,8 +392,14 @@ func NewFlowStorageManager(
 			1, 100*time.Millisecond),
 	}
 
+	// We need the client info manager to be up first
+	_, err := services.GetClientInfoManager(config_obj)
+	if err != nil {
+		return nil, err
+	}
+
 	wg.Add(1)
 	go res.houseKeeping(ctx, config_obj, wg)
 
-	return res
+	return res, nil
 }

services/orgs/services.go

@@ -505,6 +505,16 @@ func (self *OrgManager) startOrgFromContext(org_ctx *OrgContext) (err error) {
 		service_container.mu.Unlock()
 	}
 
+	if spec.ClientInfo {
+		c, err := client_info.NewClientInfoManager(ctx, wg, org_config)
+		if err != nil {
+			return err
+		}
+		service_container.mu.Lock()
+		service_container.client_info_manager = c
+		service_container.mu.Unlock()
+	}
+
 	if spec.Launcher {
 		launch, err := launcher.NewLauncherService(
 			ctx, wg, org_config)
@@ -609,21 +619,6 @@ func (self *OrgManager) startOrgFromContext(org_ctx *OrgContext) (err error) {
 		}
 	}
 
-	if spec.ClientInfo {
-		c, err := client_info.NewClientInfoManager(ctx, wg, org_config)
-		if err != nil {
-			return err
-		}
-		err = c.Start(ctx, org_config, wg)
-		if err != nil {
-			return err
-		}
-
-		service_container.mu.Lock()
-		service_container.client_info_manager = c
-		service_container.mu.Unlock()
-	}
-
 	if spec.HuntDispatcher {
 		hd, err := hunt_dispatcher.NewHuntDispatcher(
 			ctx, wg, org_config)