v0.1.0-beta.9

New Features:

- Request parameters such as query or post parameters are now added in the attack
  events and shown in the attack logs and in the event explorer pages of our dashboard. (#84)

- PII scrubbing is now performed on every event sent to Sqreen, as documented
  on https://docs.sqreen.com/guides/how-sqreen-works/#pii-scrubbing. (#86)

Fixes:

- Add PII scrubbing to the WAF logs that may include data from the request. (#87)

Internal Changes:

- The In-App WAF has been intensively optimized so that large requests can no longer impact
  its execution time. (#83)

Author: Julio Guerra

README.md

@@ -33,39 +33,41 @@ For more details, visit [sqreen.com](https://www.sqreen.com/)
 
 # Installation
 
-1. Download the Go agent and the SDK using `go get`:
+Installing the agent is just installing a regular Go package. Simply import it
+and let the toolchain do the job.
 
-    ```sh
-    $ go get -v github.com/sqreen/go-agent@v0.1.0-beta.8
-    ```
 
-1. Import the package `agent` in your `main` package of your app:
+1. Import the agent package into the `main` package of your app:
 
     ```go
     import _ "github.com/sqreen/go-agent/agent"
     ```
 
+1. Use the middleware function for the Go web framework you use:
+   - [sqhttp](https://godoc.org/github.com/sqreen/go-agent/sdk/middleware/sqhttp) for the standard `net/http` package.
+   - [Gin](https://godoc.org/github.com/sqreen/go-agent/sdk/middleware/sqgin) for `github.com/gin-gonic/gin`.
+   - [Echo](https://godoc.org/github.com/sqreen/go-agent/sdk/middleware/sqecho) for `github.com/labstack/echo`.
+   
+   If your framework is not in the list, it is usually possible to use the
+   standard `net/http` middleware. If not, please, [create an issue](http://github.com/sqreen/go-agent/issues/new)
+   to let us know.
+
 1. [Signup to Sqreen](https://my.sqreen.io/signup) to get a token for your app,
    and store it in the agent's configuration file `sqreen.yaml`:
-
+   
     ```sh
     token: your token
     app_name: Your App Name
     ```
-
+   
    This file can be stored in your current working directory when starting the
    executable, the same directory as your app's executable file, or in any other
    path by defining the configuration file location into the environment
    variable `SQREEN_CONFIG_FILE`.
 
-1. Set up Sqreen's middleware functions according to the web framework you use:
-   - [sqhttp](https://godoc.org/github.com/sqreen/go-agent/sdk/middleware/sqhttp) for the standard `net/http` package.
-   - [Gin](https://godoc.org/github.com/sqreen/go-agent/sdk/middleware/sqgin) for `github.com/gin-gonic/gin`.
-   - [Echo](https://godoc.org/github.com/sqreen/go-agent/sdk/middleware/sqecho) for `github.com/labstack/echo`.
-   
-   If your framework is not in the list, it is usually possible to use the
-   standard `net/http` middleware. If not, please open an issue in this
-   repository to start a discussion about it.
+1. You are done!  
+   Just recompile your Go program and the go toolchain will download the latests
+   agent version.
 
 1. Optionally, use the [SDK](https://godoc.org/github.com/sqreen/go-agent/sdk)
    to send security [events related to
@@ -74,6 +76,8 @@ For more details, visit [sqreen.com](https://www.sqreen.com/)
    events](https://godoc.org/github.com/sqreen/go-agent/sdk#HTTPRequestRecord.TrackEvent)
    you would like to track (eg. password changes).
 
+Find out more about the agent installation at https://docs.sqreen.com/go/installation/
+
 # Licensing
 
 Sqreen for Go is free-to-use, proprietary software.

agent/internal/actor/action_test.go

@@ -17,7 +17,7 @@ import (
 
 func TestAction(t *testing.T) {
 	t.Run("Blocking action", func(t *testing.T) {
-		action := newBlockAction(testlib.RandString(1, 20))
+		action := newBlockAction(testlib.RandPrintableUSASCIIString(1, 20))
 
 		t.Run("with duration", func(t *testing.T) {
 			t.Run("not expired", func(t *testing.T) {
@@ -26,6 +26,7 @@ func TestAction(t *testing.T) {
 			})
 			t.Run("expired", func(t *testing.T) {
 				action := withDuration(action, 0)
+				time.Sleep(time.Millisecond)
 				require.True(t, action.Expired())
 			})
 		})
@@ -44,7 +45,7 @@ func TestAction(t *testing.T) {
 			})
 
 			t.Run("Block User", func(t *testing.T) {
-				handler, err := NewUserActionHTTPHandler(action, map[string]string{"uid": testlib.RandString(1, 250)})
+				handler, err := NewUserActionHTTPHandler(action, map[string]string{"uid": testlib.RandPrintableUSASCIIString(1, 250)})
 				require.NoError(t, err)
 				require.NotNil(t, handler)
 				// Use the handler
@@ -59,13 +60,13 @@ func TestAction(t *testing.T) {
 
 	t.Run("Redirection action", func(t *testing.T) {
 		t.Run("invalid location url", func(t *testing.T) {
-			action, err := newRedirectAction(testlib.RandString(1, 20), "http//toto")
+			action, err := newRedirectAction(testlib.RandPrintableUSASCIIString(1, 20), "http//toto")
 			require.Nil(t, action)
 			require.Error(t, err)
 		})
 
 		t.Run("valid location url", func(t *testing.T) {
-			action, err := newRedirectAction(testlib.RandString(1, 20), "http://sqreen.com")
+			action, err := newRedirectAction(testlib.RandPrintableUSASCIIString(1, 20), "http://sqreen.com")
 			require.NotNil(t, action)
 			require.NoError(t, err)
 
@@ -76,6 +77,7 @@ func TestAction(t *testing.T) {
 				})
 				t.Run("expired", func(t *testing.T) {
 					action := withDuration(action, 0)
+					time.Sleep(time.Millisecond)
 					require.True(t, action.Expired())
 				})
 			})
@@ -95,7 +97,7 @@ func TestAction(t *testing.T) {
 				})
 
 				t.Run("Redirect User", func(t *testing.T) {
-					handler, err := NewUserActionHTTPHandler(action, map[string]string{"uid": testlib.RandString(1, 250)})
+					handler, err := NewUserActionHTTPHandler(action, map[string]string{"uid": testlib.RandPrintableUSASCIIString(1, 250)})
 					require.NoError(t, err)
 					require.NotNil(t, handler)
 					// Use the handler

agent/internal/actor/actor_test.go

@@ -301,7 +301,7 @@ func TestStore(t *testing.T) {
 			{
 				name: "Malformed",
 				actions: map[*api.ActionsPackResponse_Action][]net.IP{
-					NewBlockIPAction(testlib.RandString(2, 20)): nil,
+					NewBlockIPAction(testlib.RandPrintableUSASCIIString(2, 20)): nil,
 				},
 			},
 
@@ -610,8 +610,8 @@ func RandUser() map[string]string {
 	count := int(1 + rand.Uint32()%10)
 	user := make(map[string]string, count)
 	for n := 0; n < count; n++ {
-		k := testlib.RandString(1, 50)
-		v := testlib.RandString(1, 50)
+		k := testlib.RandPrintableUSASCIIString(1, 50)
+		v := testlib.RandPrintableUSASCIIString(1, 50)
 		user[k] = v
 	}
 	return user

agent/internal/agent.go

@@ -26,6 +26,7 @@ import (
 	"github.com/sqreen/go-agent/agent/internal/rule"
 	"github.com/sqreen/go-agent/agent/sqlib/sqerrors"
 	"github.com/sqreen/go-agent/agent/sqlib/sqsafe"
+	"github.com/sqreen/go-agent/agent/sqlib/sqsanitize"
 	"github.com/sqreen/go-agent/agent/sqlib/sqtime"
 	"github.com/sqreen/go-agent/agent/types"
 	"github.com/sqreen/go-agent/sdk"
@@ -139,6 +140,7 @@ type Agent struct {
 	client        *backend.Client
 	actors        *actor.Store
 	rules         *rule.Engine
+	piiScrubber   *sqsanitize.Scrubber
 }
 
 func (a *Agent) FindActionByIP(ip net.IP) (action actor.Action, exists bool, err error) {
@@ -182,6 +184,12 @@ func New(cfg *config.Config) *Agent {
 	sdkMetricsPeriod := time.Duration(cfg.SDKMetricsPeriod()) * time.Second
 	logger.Debugf("using sdk metrics store period of %s", sdkMetricsPeriod)
 
+	piiScrubber, err := sqsanitize.NewScrubber(config.ScrubberKeyRegexp, config.ScrubberValueRegexp, config.ScrubberRedactedString)
+	if err != nil {
+		logger.Error(sqerrors.Wrap(err, "ecdsa public key"))
+		return nil
+	}
+
 	// Agent graceful stopping using context cancellation.
 	ctx, cancel := context.WithCancel(context.Background())
 	return &Agent{
@@ -195,19 +203,19 @@ func New(cfg *config.Config) *Agent {
 			whitelistedIP:       metrics.NewStore("whitelisted", sdkMetricsPeriod),
 			errors:              metrics.NewStore("errors", config.ErrorMetricsPeriod),
 		},
-		ctx:     ctx,
-		cancel:  cancel,
-		config:  cfg,
-		appInfo: app.NewInfo(logger),
-		client:  backend.NewClient(cfg.BackendHTTPAPIBaseURL(), cfg, logger),
-		actors:  actor.NewStore(logger),
-		rules:   rulesEngine,
+		ctx:         ctx,
+		cancel:      cancel,
+		config:      cfg,
+		appInfo:     app.NewInfo(logger),
+		client:      backend.NewClient(cfg.BackendHTTPAPIBaseURL(), cfg, logger),
+		actors:      actor.NewStore(logger),
+		rules:       rulesEngine,
+		piiScrubber: piiScrubber,
 	}
 }
 
 func (a *Agent) NewRequestRecord(req *http.Request) (types.RequestRecord, *http.Request) {
-	rr, req := record.NewRequestRecord(a, a.logger, req, a.config)
-	return rr, req
+	return record.NewRequestRecord(a, a.logger, req, a.config)
 }
 
 func (a *Agent) Serve() error {
@@ -248,7 +256,7 @@ func (a *Agent) Serve() error {
 
 	batchSize := int(appLoginRes.Features.BatchSize)
 	if batchSize == 0 {
-		batchSize = config.MaxEventsPerHeatbeat
+		batchSize = config.EventBatchMaxEventsPerHeartbeat
 	}
 	maxStaleness := time.Duration(appLoginRes.Features.MaxStaleness) * time.Second
 	if maxStaleness == 0 {
@@ -486,10 +494,17 @@ func (m *eventManager) send(client *backend.Client, batch []Event) {
 		var event api.BatchRequest_EventFace
 		switch actual := e.(type) {
 		case *HTTPRequestRecordEvent:
-			event = (*api.RequestRecordEvent)(api.NewRequestRecordFromFace(actual))
+			event = api.RequestRecordEvent{api.NewRequestRecordFromFace(actual)}
 		case *ExceptionEvent:
 			event = api.NewExceptionEventFromFace(actual)
 		}
+
+		// Scrub the value, along with the set of scrubbed string values.
+		if _, err := m.agent.piiScrubber.Scrub(event, nil); err != nil {
+			// Only log this unexpected error and keep the event that may have been
+			// partially scrubbed.
+			m.agent.logger.Error(errors.Wrap(err, "could not send the event batch"))
+		}
 		req.Batch = append(req.Batch, *api.NewBatchRequest_EventFromFace(event))
 	}