diff --git a/accessors/overlay/fixtures/TestOverlay.golden b/accessors/overlay/fixtures/TestOverlay.golden index d64f0ec5d90..2d357a24a32 100644 --- a/accessors/overlay/fixtures/TestOverlay.golden +++ b/accessors/overlay/fixtures/TestOverlay.golden @@ -1,4 +1,5 @@ { "/file1.txt": "Hello", - "/file2.txt": "Hello Two" + "/file2.txt": "Hello Two", + "/subdir/file2.txt": "Hello Subdir" } \ No newline at end of file diff --git a/accessors/overlay/overlay_test.go b/accessors/overlay/overlay_test.go index 926f48a73d3..f64ef2c8cba 100644 --- a/accessors/overlay/overlay_test.go +++ b/accessors/overlay/overlay_test.go @@ -54,6 +54,7 @@ func (self *OverlayAccessorTestSuite) makeFile(path string, content string) { func (self *OverlayAccessorTestSuite) TestOverlay() { self.makeFile("foo1/file1.txt", "Hello") self.makeFile("foo2/file2.txt", "Hello Two") + self.makeFile("foo2/subdir/file2.txt", "Hello Subdir") scope := vql_subsystem.MakeScope(). AppendVars(ordereddict.NewDict(). @@ -71,21 +72,29 @@ func (self *OverlayAccessorTestSuite) TestOverlay() { accessor, err := accessors.GetAccessor("overlay", scope) assert.NoError(self.T(), err) - files, err := accessor.ReadDir("/") - assert.NoError(self.T(), err) - golden := ordereddict.NewDict() - for _, f := range files { - fd, err := accessor.OpenWithOSPath(f.OSPath()) + check_dir := func(file_path string) { + files, err := accessor.ReadDir(file_path) assert.NoError(self.T(), err) - data, err := utils.ReadAllWithLimit(fd, constants.MAX_MEMORY) - assert.NoError(self.T(), err) - fd.Close() + for _, f := range files { + if f.IsDir() { + continue + } + + fd, err := accessor.OpenWithOSPath(f.OSPath()) + assert.NoError(self.T(), err) + + data, err := utils.ReadAllWithLimit(fd, constants.MAX_MEMORY) + assert.NoError(self.T(), err) + fd.Close() - golden.Set(f.OSPath().String(), string(data)) + golden.Set(f.OSPath().String(), string(data)) + } } + check_dir("/") + check_dir("/subdir/") goldie.Assert(self.T(), "TestOverlay", json.MustMarshalIndent(golden)) diff --git a/bin/artifacts.go b/bin/artifacts.go index 495e64f2319..c9a99922ff1 100644 --- a/bin/artifacts.go +++ b/bin/artifacts.go @@ -160,6 +160,8 @@ func doArtifactCollect() error { return fmt.Errorf("Unable to create config: %w", err) } + config_obj.Services = services.GenericToolServices() + ctx, top_cancel := install_sig_handler() defer top_cancel() diff --git a/bin/glob.go b/bin/glob.go new file mode 100644 index 00000000000..9153263d8cc --- /dev/null +++ b/bin/glob.go @@ -0,0 +1,20 @@ +package main + +import "path/filepath" + +func expandOneGlob(path string) []string { + res, err := filepath.Glob(path) + if err != nil { + return []string{path} + } + return res +} + +// Needed for Windows as the shell does not expand globs. +func expandGlobs(paths []string) (res []string) { + for _, p := range paths { + res = append(res, expandOneGlob(p)...) + } + + return res +} diff --git a/bin/reformat.go b/bin/reformat.go index 3fa1681cfb1..0b496ab85a7 100644 --- a/bin/reformat.go +++ b/bin/reformat.go @@ -43,7 +43,7 @@ func doReformat() error { logger := logging.GetLogger(config_obj, &logging.ToolComponent) var artifact_paths []string - for _, artifact_path := range *reformat_args { + for _, artifact_path := range expandGlobs(*reformat_args) { abs, err := filepath.Abs(artifact_path) if err != nil { logger.Error("reformat: could not get absolute path for %v", artifact_path) diff --git a/bin/verify.go b/bin/verify.go index 627fa8dd16f..44687da6b63 100644 --- a/bin/verify.go +++ b/bin/verify.go @@ -52,7 +52,7 @@ func doVerify() error { var artifact_paths []string - for _, artifact_path := range *verify_args { + for _, artifact_path := range expandGlobs(*verify_args) { abs, err := filepath.Abs(artifact_path) if err != nil { logger.Error("verify: could not get absolute path for %v", artifact_path) diff --git a/docs/references/vql.yaml b/docs/references/vql.yaml index 49b10c96771..4241ab99001 100644 --- a/docs/references/vql.yaml +++ b/docs/references/vql.yaml @@ -8288,7 +8288,6 @@ FROM range(start=0, end=20) }, exit='x=>x._value >= my_limit', inherit=true) ``` - type: Plugin args: - name: query @@ -12507,6 +12506,35 @@ - linux_amd64_cgo - windows_386_cgo - windows_amd64_cgo +- name: write_file + description: | + Writes a string onto a file. + + This VQL function is a convenience wrapper to the copy() function. + + type: Function + args: + - name: data + type: string + description: The data to write + required: true + - name: dest + type: string + description: The destination file to write. + required: true + - name: permissions + type: string + description: Required permissions (e.g. 'x'). + - name: append + type: bool + description: If true we append to the target file otherwise truncate it + - name: create_directories + type: bool + description: If true we ensure the destination directories exist + metadata: + permissions: FILESYSTEM_WRITE,FILESYSTEM_READ + platforms: + - linux_amd64_cgo - name: write_jsonl description: Write a query into a JSONL file. type: Plugin diff --git a/services/client_info/client_info.go b/services/client_info/client_info.go index ea8dfa5edaf..4707e5597d0 100644 --- a/services/client_info/client_info.go +++ b/services/client_info/client_info.go @@ -583,7 +583,11 @@ func (self *ClientInfoManager) Set( func NewClientInfoManager( ctx context.Context, wg *sync.WaitGroup, - config_obj *config_proto.Config) (*ClientInfoManager, error) { + config_obj *config_proto.Config) (services.ClientInfoManager, error) { + + if config_obj.Datastore == nil { + return &DummyClientInfoManager{}, nil + } // Calculate a unique id for each service. service := &ClientInfoManager{ @@ -614,8 +618,6 @@ func NewClientInfoManager( <-ctx.Done() - utils.DlvBreak() - // When we shut down make sure to save the snapshot. subctx, cancel := utils.WithTimeoutCause( context.Background(), 100*time.Second, @@ -629,6 +631,11 @@ func NewClientInfoManager( } }() + err = service.Start(ctx, config_obj, wg) + if err != nil { + return nil, err + } + return service, nil } diff --git a/services/client_info/client_info_test.go b/services/client_info/client_info_test.go index 008f64a9eb5..00f5f62b2ec 100644 --- a/services/client_info/client_info_test.go +++ b/services/client_info/client_info_test.go @@ -160,7 +160,7 @@ func (self *ClientInfoTestSuite) TestMasterMinion() { self.Ctx, self.Sm.Wg, minion_config) assert.NoError(self.T(), err) - err = minion_client_info_manager.Start( + err = minion_client_info_manager.(*client_info.ClientInfoManager).Start( self.Ctx, minion_config, self.Sm.Wg) assert.NoError(self.T(), err) diff --git a/services/client_info/dummy.go b/services/client_info/dummy.go new file mode 100644 index 00000000000..37909a87228 --- /dev/null +++ b/services/client_info/dummy.go @@ -0,0 +1,110 @@ +package client_info + +import ( + "context" + + "github.com/Velocidex/ordereddict" + crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" + "www.velocidex.com/golang/velociraptor/services" + "www.velocidex.com/golang/velociraptor/utils" +) + +type DummyClientInfoManager struct{} + +func (self DummyClientInfoManager) ListClients(ctx context.Context) <-chan string { + output_chan := make(chan string) + close(output_chan) + return output_chan +} + +// Used to set a new client record. To modify an existing record - +// or set a new one use Modify() +func (self DummyClientInfoManager) Set(ctx context.Context, + client_info *services.ClientInfo) error { + return utils.NotImplementedError +} + +// Modify a record or set a new one - if the record is not found, +// modifier will receive a nil client_info. The ClientInfoManager +// can not be accessed within the modifier function as it is +// locked for the duration of the change. +func (self DummyClientInfoManager) Modify(ctx context.Context, client_id string, + modifier func(client_info *services.ClientInfo) ( + new_record *services.ClientInfo, err error)) error { + return utils.NotImplementedError +} + +func (self DummyClientInfoManager) Get(ctx context.Context, + client_id string) (*services.ClientInfo, error) { + return nil, utils.NotImplementedError +} + +func (self DummyClientInfoManager) Remove(ctx context.Context, client_id string) {} + +func (self DummyClientInfoManager) GetStats( + ctx context.Context, client_id string) (*services.Stats, error) { + return nil, utils.NotImplementedError +} + +func (self DummyClientInfoManager) UpdateStats( + ctx context.Context, client_id string, stats *services.Stats) error { + return utils.NotImplementedError +} + +// Get the client's tasks and remove them from the queue. +func (self DummyClientInfoManager) GetClientTasks(ctx context.Context, + client_id string) ([]*crypto_proto.VeloMessage, error) { + return nil, utils.NotImplementedError +} + +// Get all the tasks without de-queuing them. +func (self DummyClientInfoManager) PeekClientTasks(ctx context.Context, + client_id string) ([]*crypto_proto.VeloMessage, error) { + return nil, utils.NotImplementedError +} + +func (self DummyClientInfoManager) QueueMessagesForClient( + ctx context.Context, + client_id string, + req []*crypto_proto.VeloMessage, + notify bool, /* Also notify the client about the new task */ +) error { + return utils.NotImplementedError +} + +func (self DummyClientInfoManager) QueueMessageForClient( + ctx context.Context, + client_id string, + req *crypto_proto.VeloMessage, + notify bool, /* Also notify the client about the new task */ + completion func()) error { + return utils.NotImplementedError +} + +func (self DummyClientInfoManager) UnQueueMessageForClient( + ctx context.Context, + client_id string, + req *crypto_proto.VeloMessage) error { + return utils.NotImplementedError +} + +// Be able to manipulate the client and server metadata. +func (self DummyClientInfoManager) GetMetadata(ctx context.Context, + client_id string) (*ordereddict.Dict, error) { + return nil, utils.NotImplementedError +} + +func (self DummyClientInfoManager) SetMetadata(ctx context.Context, + client_id string, metadata *ordereddict.Dict, principal string) error { + return utils.NotImplementedError +} + +func (self DummyClientInfoManager) ValidateClientId(client_id string) error { + return utils.NotImplementedError +} + +func (self DummyClientInfoManager) DeleteClient( + ctx context.Context, client_id, principal string, + progress chan services.DeleteFlowResponse, really_do_it bool) error { + return utils.NotImplementedError +} diff --git a/services/launcher.go b/services/launcher.go index 68548842bd2..f45d33ab0d8 100644 --- a/services/launcher.go +++ b/services/launcher.go @@ -84,13 +84,6 @@ func GetLauncher(config_obj *config_proto.Config) (Launcher, error) { } svc := org_manager.Services(config_obj.OrgId) - - // We need the client info manager to be up first - _, err = svc.ClientInfoManager() - if err != nil { - return nil, err - } - return svc.Launcher() } diff --git a/services/launcher/launcher.go b/services/launcher/launcher.go index 1f38849dc7c..d4868d392e4 100644 --- a/services/launcher/launcher.go +++ b/services/launcher/launcher.go @@ -826,8 +826,12 @@ func NewLauncherService( return &Launcher{Storage_: &DummyStorer{}}, nil } - res := &Launcher{ - Storage_: NewFlowStorageManager(ctx, config_obj, wg), + storage, err := NewFlowStorageManager(ctx, config_obj, wg) + if err != nil { + return nil, err } - return res, nil + + return &Launcher{ + Storage_: storage, + }, nil } diff --git a/services/launcher/storage.go b/services/launcher/storage.go index 2e869d507cf..f70a51adb31 100644 --- a/services/launcher/storage.go +++ b/services/launcher/storage.go @@ -380,7 +380,7 @@ func (self *FlowStorageManager) GetFlowRequests( func NewFlowStorageManager( ctx context.Context, config_obj *config_proto.Config, - wg *sync.WaitGroup) *FlowStorageManager { + wg *sync.WaitGroup) (*FlowStorageManager, error) { res := &FlowStorageManager{ indexBuilders: make(map[string]*flowIndexBuilder), throttler: utils.NewThrottlerWithDuration(time.Second), @@ -392,8 +392,14 @@ func NewFlowStorageManager( 1, 100*time.Millisecond), } + // We need the client info manager to be up first + _, err := services.GetClientInfoManager(config_obj) + if err != nil { + return nil, err + } + wg.Add(1) go res.houseKeeping(ctx, config_obj, wg) - return res + return res, nil } diff --git a/services/orgs/services.go b/services/orgs/services.go index 49d5de0fcc5..ac2294e31a3 100644 --- a/services/orgs/services.go +++ b/services/orgs/services.go @@ -505,6 +505,16 @@ func (self *OrgManager) startOrgFromContext(org_ctx *OrgContext) (err error) { service_container.mu.Unlock() } + if spec.ClientInfo { + c, err := client_info.NewClientInfoManager(ctx, wg, org_config) + if err != nil { + return err + } + service_container.mu.Lock() + service_container.client_info_manager = c + service_container.mu.Unlock() + } + if spec.Launcher { launch, err := launcher.NewLauncherService( ctx, wg, org_config) @@ -609,21 +619,6 @@ func (self *OrgManager) startOrgFromContext(org_ctx *OrgContext) (err error) { } } - if spec.ClientInfo { - c, err := client_info.NewClientInfoManager(ctx, wg, org_config) - if err != nil { - return err - } - err = c.Start(ctx, org_config, wg) - if err != nil { - return err - } - - service_container.mu.Lock() - service_container.client_info_manager = c - service_container.mu.Unlock() - } - if spec.HuntDispatcher { hd, err := hunt_dispatcher.NewHuntDispatcher( ctx, wg, org_config) diff --git a/services/spec.go b/services/spec.go index 9fac4cbde7e..e15bbbbf878 100644 --- a/services/spec.go +++ b/services/spec.go @@ -19,6 +19,11 @@ func GenericToolServices() *config_proto.ServerServicesConfig { JournalService: true, UserManager: true, NotificationService: true, + + // If the config provides a datastore we can use the real + // Client Info Manager, otherwise we will use a dummy + // one. This is mostly used by VQL functions that may need it. + ClientInfo: true, } } diff --git a/vql/filesystem/copy.go b/vql/filesystem/copy.go index ec30b239c38..0a42d00d350 100644 --- a/vql/filesystem/copy.go +++ b/vql/filesystem/copy.go @@ -47,7 +47,7 @@ type CopyFunctionArgs struct { type CopyFunction struct{} -func (self *CopyFunction) Call(ctx context.Context, +func (self CopyFunction) Call(ctx context.Context, scope vfilter.Scope, args *ordereddict.Dict) vfilter.Any { defer vql_subsystem.RegisterMonitor(ctx, "copy", args)() @@ -169,5 +169,5 @@ func (self CopyFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *v } func init() { - vql_subsystem.RegisterFunction(&CopyFunction{}) + vql_subsystem.RegisterFunction(CopyFunction{}) } diff --git a/vql/filesystem/write_file.go b/vql/filesystem/write_file.go new file mode 100644 index 00000000000..b94ef88243b --- /dev/null +++ b/vql/filesystem/write_file.go @@ -0,0 +1,58 @@ +package filesystem + +import ( + "context" + + "github.com/Velocidex/ordereddict" + "www.velocidex.com/golang/velociraptor/acls" + "www.velocidex.com/golang/velociraptor/vql" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + vfilter "www.velocidex.com/golang/vfilter" + "www.velocidex.com/golang/vfilter/arg_parser" +) + +type WriteFunctionArgs struct { + Data string `vfilter:"required,field=data,doc=The data to write"` + Destination string `vfilter:"required,field=dest,doc=The destination file to write."` + Permissions string `vfilter:"optional,field=permissions,doc=Required permissions (e.g. 'x')."` + Append bool `vfilter:"optional,field=append,doc=If true we append to the target file otherwise truncate it"` + Directories bool `vfilter:"optional,field=create_directories,doc=If true we ensure the destination directories exist"` +} + +type WriteFunction struct{} + +// This is basically an alias to the copy() VQL function so we just delegate to that +func (self WriteFunction) Call(ctx context.Context, + scope vfilter.Scope, + args *ordereddict.Dict) vfilter.Any { + defer vql_subsystem.RegisterMonitor(ctx, "write_file", args)() + + arg := &WriteFunctionArgs{} + err := arg_parser.ExtractArgsWithContext(ctx, scope, args, arg) + if err != nil { + scope.Log("write_file: %v", err) + return vfilter.Null{} + } + + return CopyFunction{}.Call(ctx, scope, ordereddict.NewDict(). + Set("accessor", "data"). + Set("filename", arg.Data). + Set("dest", arg.Destination). + Set("permissions", arg.Permissions). + Set("append", arg.Append). + Set("create_directories", arg.Directories)) +} + +func (self WriteFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo { + return &vfilter.FunctionInfo{ + Name: "write_file", + Doc: "Writes a string onto a file.", + ArgType: type_map.AddType(scope, &WriteFunctionArgs{}), + Metadata: vql.VQLMetadata().Permissions( + acls.FILESYSTEM_WRITE, acls.FILESYSTEM_READ).Build(), + } +} + +func init() { + vql_subsystem.RegisterFunction(WriteFunction{}) +}