allow watcher to self-delete on shutdown (#3565)

* allow watcher to self-delete on shutdown

* rename lapi endpoing

* rename config option

* test

* fix tests

* docker: $UNREGISTER_ON_EXIT

* docker test

* update test dependencies, with custom "stop_timeout"

* update url in client

* meh

* wip

* url

---------

Co-authored-by: Thibault "bui" Koechlin <[email protected]>
Co-authored-by: marco <[email protected]>

Author: 3 people

cmd/crowdsec/serve.go

@@ -15,6 +15,7 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/csdaemon"
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
@@ -322,6 +323,18 @@ func HandleSignals(cConfig *csconfig.Config) error {
 	if err == nil {
 		log.Warning("Crowdsec service shutting down")
 	}
+	if cConfig.API != nil && cConfig.API.Client != nil && cConfig.API.Client.UnregisterOnExit {
+		log.Warning("Unregistering watcher")
+		lapiClient, err := apiclient.GetLAPIClient()
+		if err != nil {
+			return err
+		}
+		_, err = lapiClient.Auth.UnregisterWatcher(context.TODO())
+		if err != nil {
+			return fmt.Errorf("failed to unregister watcher: %w", err)
+		}
+		log.Warning("Watcher unregistered")
+	}
 
 	return err
 }

docker/README.md

@@ -289,6 +289,7 @@ config.yaml) each time the container is run.
 | __Agent__               | | (these don't work with DISABLE_AGENT) |
 | `TYPE`                  | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
 | `DSN`                   | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
+| `UNREGISTER_ON_EXIT`    | | Remove the agent from the LAPI when its container is stopped. |
 |                         | | |
 | __Bouncers__            | | |
 | `BOUNCER_KEY_<name>`    | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |

docker/docker_start.sh

@@ -474,6 +474,8 @@ else
     conf_set '.api.server.enable=true'
 fi
 
+conf_set_if "$UNREGISTER_ON_EXIT" '.api.client.unregister_on_exit=env(UNREGISTER_ON_EXIT)'
+
 ARGS=""
 if [ "$CONFIG_FILE" != "" ]; then
     ARGS="-c $CONFIG_FILE"

docker/test/pyproject.toml

@@ -6,7 +6,7 @@ readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
     "pytest>=8.3.4",
-    "pytest-cs",
+    "pytest-cs>=0.7.22",
     "pytest-dotenv>=0.5.2",
     "pytest-xdist>=3.6.1",
 ]

docker/test/tests/test_agent_only.py

@@ -1,4 +1,5 @@
 import secrets
+import time
 from http import HTTPStatus
 
 import pytest
@@ -35,3 +36,52 @@ def test_split_lapi_agent(crowdsec, flavor: str) -> None:
             assert res.exit_code == 0
             stdout = res.output.decode()
             assert "You can successfully interact with Local API (LAPI)" in stdout
+
+
+def test_unregister_on_exit(crowdsec, flavor: str) -> None:
+    rand = str(secrets.randbelow(10000))
+    lapiname = f"lapi-{rand}"
+    agentname = f"agent-{rand}"
+
+    lapi_env = {
+        "AGENT_USERNAME": "testagent",
+        "AGENT_PASSWORD": "testpassword",
+    }
+
+    agent_env = {
+        "AGENT_USERNAME": "testagent",
+        "AGENT_PASSWORD": "testpassword",
+        "DISABLE_LOCAL_API": "true",
+        "LOCAL_API_URL": f"http://{lapiname}:8080",
+        "UNREGISTER_ON_EXIT": "true",
+    }
+
+    cs_lapi = crowdsec(name=lapiname, environment=lapi_env, flavor=flavor)
+    cs_agent = crowdsec(name=agentname, environment=agent_env, flavor=flavor, stop_timeout=5)
+
+    with cs_lapi as lapi:
+        lapi.wait_for_log("*CrowdSec Local API listening on *:8080*")
+        lapi.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
+
+        res = lapi.cont.exec_run("cscli machines list")
+        assert res.exit_code == 0
+        # the machine is created in the lapi entrypoint
+        assert "testagent" in res.output.decode()
+
+        with cs_agent as agent:
+            agent.wait_for_log("*Starting processing data*")
+            res = agent.cont.exec_run("cscli lapi status")
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout
+
+            res = lapi.cont.exec_run("cscli machines list")
+            assert res.exit_code == 0
+            assert "testagent" in res.output.decode()
+
+        time.sleep(2)
+
+        res = lapi.cont.exec_run("cscli machines list")
+        assert res.exit_code == 0
+        # and it's not there anymore
+        assert "testagent" not in res.output.decode()

docker/test/uv.lock

@@ -19,7 +19,7 @@ type enrollRequest struct {
 }
 
 func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) {
-	u := fmt.Sprintf("%s/watchers", s.client.URLPrefix)
+	u := fmt.Sprintf("%s/watchers/self", s.client.URLPrefix)
 
 	req, err := s.client.PrepareRequest(ctx, http.MethodDelete, u, nil)
 	if err != nil {

pkg/apiclient/auth_service.go

@@ -188,7 +188,7 @@ func TestWatcherUnregister(t *testing.T) {
 	defer teardown()
 	// body: models.WatcherRegistrationRequest{MachineID: &config.MachineID, Password: &config.Password}
 
-	mux.HandleFunc("/watchers", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/watchers/self", func(w http.ResponseWriter, r *http.Request) {
 		testMethod(t, r, "DELETE")
 		assert.Equal(t, int64(0), r.ContentLength)
 		w.WriteHeader(http.StatusOK)

pkg/apiclient/auth_service_test.go

@@ -130,6 +130,7 @@ func (c *Controller) NewV1() error {
 		jwtAuth.GET("/allowlists/check/:ip_or_range", c.HandlerV1.CheckInAllowlist)
 		jwtAuth.HEAD("/allowlists/check/:ip_or_range", c.HandlerV1.CheckInAllowlist)
 		jwtAuth.POST("/allowlists/check", c.HandlerV1.CheckInAllowlistBulk)
+		jwtAuth.DELETE("/watchers/self", c.HandlerV1.DeleteMachine)
 	}
 
 	apiKeyAuth := groupV1.Group("")

pkg/apiserver/controllers/controller.go

@@ -80,3 +80,27 @@ func (c *Controller) CreateMachine(gctx *gin.Context) {
 		gctx.Status(http.StatusCreated)
 	}
 }
+
+func (c *Controller) DeleteMachine(gctx *gin.Context) {
+	ctx := gctx.Request.Context()
+
+	machineID, err := getMachineIDFromContext(gctx)
+
+	if err != nil {
+		gctx.JSON(http.StatusBadRequest, gin.H{"message": err.Error()})
+		return
+	}
+	if machineID == "" {
+		gctx.JSON(http.StatusBadRequest, gin.H{"message": "machineID not found in claims"})
+		return
+	}
+
+	if err := c.DBClient.DeleteWatcher(ctx, machineID); err != nil {
+		c.HandleDBErrors(gctx, err)
+		return
+	}
+
+	log.WithFields(log.Fields{"ip": gctx.ClientIP(), "machine_id": machineID}).Info("Deleted machine")
+
+	gctx.Status(http.StatusNoContent)
+}