Skip to content

Commit 20053b7

Browse files
authored
cscli setup: new service detection and configuration (#3730)
1 parent 57989ea commit 20053b7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+5011
-4620
lines changed

.github/workflows/docker-tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
flavor: ["slim", "debian"]
2525

2626
runs-on: ubuntu-latest
27-
timeout-minutes: 30
27+
timeout-minutes: 20
2828
steps:
2929

3030
- name: Check out the repo

Dockerfile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,11 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 CGO_CFLAGS="-D_LARGEFILE64_
2525
cd - >/dev/null && \
2626
cscli hub update --with-content && \
2727
cscli collections install crowdsecurity/linux && \
28-
cscli parsers install crowdsecurity/whitelists
28+
cscli parsers install crowdsecurity/whitelists && \
29+
echo '{"source": "file", "filename": "/does/not/exist", "labels": {"type": "syslog"}}' > /etc/crowdsec/acquis.yaml
30+
31+
# we create a useless acquis.yaml, which will be overridden by a mounted volume
32+
# in most cases, but is still required for the container to start during tests
2933

3034
# In case we need to remove agents here..
3135
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete

Dockerfile.debian

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,12 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
3030
cd - >/dev/null && \
3131
cscli hub update --with-content && \
3232
cscli collections install crowdsecurity/linux && \
33-
cscli parsers install crowdsecurity/whitelists
33+
cscli parsers install crowdsecurity/whitelists && \
34+
echo '{"source": "file", "filename": "/does/not/exist", "labels": {"type": "syslog"}}' > /etc/crowdsec/acquis.yaml
35+
36+
# we create a useless acquis.yaml, which will be overridden by a mounted volume
37+
# in most cases, but is still required for the container to start during tests
38+
3439

3540
# In case we need to remove agents here..
3641
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete

cmd/crowdsec-cli/clihub/hub.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package clihub
33
import (
44
"context"
55
"encoding/json"
6+
"errors"
67
"fmt"
78
"io"
89
"os"
@@ -44,6 +45,10 @@ The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](htt
4445
cscli hub update
4546
cscli hub upgrade`,
4647
DisableAutoGenTag: true,
48+
Args: args.NoArgs,
49+
RunE: func(cmd *cobra.Command, _ []string) error {
50+
return cmd.Usage()
51+
},
4752
}
4853

4954
cmd.AddCommand(cli.newBranchCmd())
@@ -228,7 +233,12 @@ func (cli *cliHub) upgrade(ctx context.Context, interactive bool, dryRun bool, f
228233
showPlan := (log.StandardLogger().Level >= log.InfoLevel)
229234
verbosePlan := (cfg.Cscli.Output == "raw")
230235

231-
if err := plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan); err != nil {
236+
err = plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan)
237+
switch {
238+
case errors.Is(err, hubops.ErrUserCanceled):
239+
// not a real error, and we'll want to print the reload message anyway
240+
fmt.Fprintln(os.Stdout, err.Error())
241+
case err != nil:
232242
return err
233243
}
234244

cmd/crowdsec-cli/cliitem/cmdinstall.go

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,15 @@ func (cli *cliItem) install(ctx context.Context, args []string, interactive bool
8686
showPlan := (log.StandardLogger().Level >= log.InfoLevel)
8787
verbosePlan := (cfg.Cscli.Output == "raw")
8888

89-
if err := plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan); err != nil {
90-
if !ignoreError {
91-
return err
92-
}
93-
89+
err = plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan)
90+
switch {
91+
case errors.Is(err, hubops.ErrUserCanceled):
92+
// not a real error, and we'll want to print the reload message anyway
93+
fmt.Fprintln(os.Stdout, err.Error())
94+
case ignoreError:
9495
log.Error(err)
96+
case err != nil:
97+
return err
9598
}
9699

97100
if msg := reload.UserMessage(); msg != "" && plan.ReloadNeeded {

cmd/crowdsec-cli/cliitem/cmdremove.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,12 @@ func (cli *cliItem) remove(ctx context.Context, args []string, interactive bool,
102102
showPlan := (log.StandardLogger().Level >= log.InfoLevel)
103103
verbosePlan := (cfg.Cscli.Output == "raw")
104104

105-
if err := plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan); err != nil {
105+
err = plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan)
106+
switch {
107+
case errors.Is(err, hubops.ErrUserCanceled):
108+
// not a real error, and we'll want to print the reload message anyway
109+
fmt.Fprintln(os.Stdout, err.Error())
110+
case err != nil:
106111
return err
107112
}
108113

cmd/crowdsec-cli/cliitem/cmdupgrade.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package cliitem
33
import (
44
"cmp"
55
"context"
6+
"errors"
67
"fmt"
78
"os"
89

@@ -67,7 +68,12 @@ func (cli *cliItem) upgrade(ctx context.Context, args []string, interactive bool
6768
showPlan := (log.StandardLogger().Level >= log.InfoLevel)
6869
verbosePlan := (cfg.Cscli.Output == "raw")
6970

70-
if err := plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan); err != nil {
71+
err = plan.Execute(ctx, interactive, dryRun, showPlan, verbosePlan)
72+
switch {
73+
case errors.Is(err, hubops.ErrUserCanceled):
74+
// not a real error, and we'll want to print the reload message anyway
75+
fmt.Fprintln(os.Stdout, err.Error())
76+
case err != nil:
7177
return err
7278
}
7379

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
package clisetup
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"os"
7+
"strings"
8+
9+
"github.com/AlecAivazis/survey/v2"
10+
"github.com/fatih/color"
11+
"github.com/hexops/gotextdiff"
12+
"github.com/hexops/gotextdiff/myers"
13+
"github.com/hexops/gotextdiff/span"
14+
"github.com/spf13/cobra"
15+
16+
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args"
17+
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisetup/setup"
18+
)
19+
20+
type acquisitionFlags struct {
21+
acquisDir string
22+
}
23+
24+
func (f *acquisitionFlags) bind(cmd *cobra.Command) {
25+
flags := cmd.Flags()
26+
flags.StringVar(&f.acquisDir, "acquis-dir", "", "Directory for the acquisition configuration")
27+
}
28+
29+
func (cli *cliSetup) newInstallAcquisitionCmd() *cobra.Command {
30+
var dryRun bool
31+
32+
f := acquisitionFlags{}
33+
34+
cmd := &cobra.Command{
35+
Use: "install-acquisition [setup_file]",
36+
Short: "Generate acquisition configuration from a setup file",
37+
Long: `Generate acquisition configuration from a setup file.
38+
39+
This command reads a setup.yaml specification (typically generated by 'cscli setup detect')
40+
and creates one acquisition file for each listed service.
41+
By default the files are placed in the acquisition directory,
42+
which you can override with --acquis-dir.`,
43+
Example: `# detect running services, create a setup file
44+
cscli setup detect > setup.yaml
45+
46+
# write configuration files in acquis.d
47+
cscli setup install-acquisition setup.yaml
48+
49+
# write files to a specific directory
50+
cscli setup install-acquisition --acquis-dir /path/to/acquis.d
51+
52+
# dry-run to preview what would be created
53+
cscli setup install-acquisition setup.yaml --dry-run
54+
`,
55+
Args: args.ExactArgs(1),
56+
DisableAutoGenTag: true,
57+
RunE: func(cmd *cobra.Command, args []string) error {
58+
inputReader, err := maybeStdinFile(args[0])
59+
if err != nil {
60+
return err
61+
}
62+
63+
stup, err := setup.ParseSetupYAML(inputReader, true, cli.cfg().Cscli.Color != "no")
64+
if err != nil {
65+
return err
66+
}
67+
68+
return cli.acquisition(stup.CollectAcquisitionSpecs(), f.acquisDir, false, dryRun)
69+
},
70+
}
71+
72+
f.bind(cmd)
73+
74+
flags := cmd.Flags()
75+
flags.BoolVar(&dryRun, "dry-run", false, "simulate the installation without making any changes")
76+
77+
return cmd
78+
}
79+
80+
func colorizeDiff(diff string) string {
81+
var b strings.Builder
82+
83+
for line := range strings.SplitSeq(diff, "\n") {
84+
switch {
85+
case strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++"):
86+
b.WriteString(color.GreenString(line) + "\n")
87+
case strings.HasPrefix(line, "-") && !strings.HasPrefix(line, "---"):
88+
b.WriteString(color.RedString(line) + "\n")
89+
case strings.HasPrefix(line, "@@"):
90+
b.WriteString(color.New(color.Bold, color.FgCyan).Sprint(line) + "\n")
91+
default:
92+
b.WriteString(line + "\n")
93+
}
94+
}
95+
96+
return b.String()
97+
}
98+
99+
func shouldOverwrite(path string, newContent []byte) (bool, error) {
100+
oldContent, err := os.ReadFile(path)
101+
if err != nil {
102+
// File doesn't exist or unreadable, assume overwrite
103+
return true, nil //nolint: nilerr
104+
}
105+
106+
if err := VerifyChecksum(bytes.NewReader(oldContent)); err == nil {
107+
// Valid checksum, safe to overwrite silently
108+
return true, nil
109+
}
110+
111+
// Invalid or missing checksum, show diff and ask
112+
edits := myers.ComputeEdits(span.URIFromPath(path), string(oldContent), string(newContent))
113+
diff := gotextdiff.ToUnified(path, path, string(oldContent), edits)
114+
115+
fmt.Fprintln(os.Stdout, color.YellowString("This file was modified after being generated by 'cscli setup'. The following changes will be made:\n"))
116+
117+
fmt.Fprint(os.Stdout, colorizeDiff(fmt.Sprintf("%s", diff)))
118+
119+
var overwrite bool
120+
121+
prompt := &survey.Confirm{
122+
Message: "Do you want to overwrite with the new version?",
123+
Default: false,
124+
}
125+
126+
if err := survey.AskOne(prompt, &overwrite); err != nil {
127+
return false, fmt.Errorf("prompt failed: %w", err)
128+
}
129+
130+
fmt.Fprintln(os.Stdout)
131+
132+
return overwrite, nil
133+
}
134+
135+
// processAcquisitionSpec handles the creation of a single acquisition file.
136+
//
137+
// It includes an header with the appropriate checksum.
138+
// In dry-run, prints the content to stdout instead of writing to disk.
139+
// In interactive mode, it prompts the user before overwriting an existing file unless it's pristine.
140+
func (cli *cliSetup) processAcquisitionSpec(spec setup.AcquisitionSpec, toDir string, interactive, dryRun bool) error {
141+
if spec.Datasource == nil {
142+
return nil
143+
}
144+
145+
path, err := spec.Path(toDir)
146+
if err != nil {
147+
return err
148+
}
149+
150+
content, err := spec.ToYAML()
151+
if err != nil {
152+
return err
153+
}
154+
155+
if dryRun {
156+
_, _ = fmt.Fprintln(os.Stdout, "(dry run) "+path+"\n"+color.BlueString(string(content)))
157+
return nil
158+
}
159+
160+
contentWithHeader := spec.AddHeader(content)
161+
162+
if interactive {
163+
ok, err := shouldOverwrite(path, contentWithHeader)
164+
if err != nil {
165+
return err
166+
}
167+
168+
if !ok {
169+
fmt.Fprintln(os.Stdout, "skipped "+path)
170+
return nil
171+
}
172+
}
173+
174+
fmt.Fprintln(os.Stdout, "creating "+path)
175+
176+
writer, err := spec.Open(toDir)
177+
if err != nil {
178+
return err
179+
}
180+
defer writer.Close()
181+
182+
_, err = writer.Write(contentWithHeader)
183+
if err != nil {
184+
return fmt.Errorf("writing acquisition to %q: %w", path, err)
185+
}
186+
187+
return nil
188+
}
189+
190+
func (cli *cliSetup) acquisition(acquisitionSpecs []setup.AcquisitionSpec, toDir string, interactive bool, dryRun bool) error {
191+
cfg := cli.cfg()
192+
193+
if toDir == "" {
194+
toDir = cfg.Crowdsec.AcquisitionDirPath
195+
}
196+
197+
if toDir == "" {
198+
return fmt.Errorf("no acquisition directory specified, please use --acquis-dir or set crowdsec_services.acquisition_dir in %q", cfg.FilePath)
199+
}
200+
201+
for _, spec := range acquisitionSpecs {
202+
if err := cli.processAcquisitionSpec(spec, toDir, interactive, dryRun); err != nil {
203+
return err
204+
}
205+
}
206+
207+
return nil
208+
}

0 commit comments

Comments
 (0)