Feature/customize log and lift test coverage (#28)

* complete the structure of influxlog

* add cache tests

* refactor influx db writer

Author: sillygod

backends/redis_test.go

@@ -52,7 +52,8 @@ func (suite *RedisBackendTestSuite) TestWriteCacheInRedis() {
 
 	suite.Equal(len(content), backend.Length())
 
-	backend.Close()
+	err = backend.Close()
+	suite.Nil(err)
 
 	reader, err := backend.GetReader()
 	suite.Nil(err)

cache_test.go

@@ -1,11 +1,13 @@
 package httpcache
 
 import (
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
 
+	"github.com/sillygod/cdp-cache/backends"
 	"github.com/stretchr/testify/suite"
 )
 
@@ -16,10 +18,10 @@ func makeRequest(url string, headers http.Header) *http.Request {
 }
 
 func makeResponse(code int, headers http.Header) *Response {
-	return &Response{
-		Code:       code,
-		snapHeader: headers,
-	}
+	response := NewResponse()
+	response.Code = code
+	response.snapHeader = headers
+	return response
 }
 
 func makeHeader(key string, value string) http.Header {
@@ -142,7 +144,81 @@ func (suite *RuleMatcherTestSuite) TestHeaderNotMatched() {
 	suite.False(match)
 }
 
+type EntryTestSuite struct {
+	suite.Suite
+	config *Config
+}
+
+func (suite *EntryTestSuite) SetupSuite() {
+	err := backends.InitGroupCacheRes(50 * 1024 * 1024)
+	suite.Nil(err)
+	suite.config = getDefaultConfig()
+}
+
+func (suite *EntryTestSuite) TestEntryWritePublicResponse() {
+
+	req := makeRequest("/", http.Header{})
+	res := makeResponse(200, makeHeader("Cache-Control", "max-age=43200"))
+	entry := NewEntry("unique_key", req, res, suite.config)
+
+	suite.Equal("unique_key", entry.Key())
+	rw := httptest.NewRecorder()
+
+	suite.config.Type = inMemory
+	input := []byte(`rain cats and dogs`)
+
+	go func() {
+		entry.Response.Write(input)
+		entry.Response.Close() // we can write the entry's body to rw after closing upstream response
+	}()
+
+	entry.setBackend(req.Context(), suite.config)
+
+	entry.WriteBodyTo(rw)
+	result, err := ioutil.ReadAll(rw.Result().Body)
+	suite.Nil(err)
+	suite.Equal(input, result)
+}
+
+func (suite *EntryTestSuite) TestEntryWritePrivateResponse() {
+	req := makeRequest("/", http.Header{})
+	res := makeResponse(502, http.Header{})
+	entry := NewEntry("unique_key2", req, res, suite.config)
+
+	go func() {
+		entry.Response.Close()
+	}()
+
+	rw := httptest.NewRecorder()
+	entry.WriteBodyTo(rw)
+	suite.Equal(502, rw.Code)
+}
+
+func (suite *EntryTestSuite) TearDownSuite() {
+	err := backends.ReleaseGroupCacheRes()
+	suite.Nil(err)
+}
+
+type HTTPCacheTestSuite struct {
+	suite.Suite
+	config *Config
+}
+
+func (suite *HTTPCacheTestSuite) SetupSuite() {
+	err := backends.InitGroupCacheRes(50 * 1024 * 1024)
+	suite.Nil(err)
+	suite.config = getDefaultConfig()
+}
+
+// TODO: add http cache test
+
+func (suite *HTTPCacheTestSuite) TearDownSuite() {
+	err := backends.ReleaseGroupCacheRes()
+	suite.Nil(err)
+}
+
 func TestCacheStatusTestSuite(t *testing.T) {
 	suite.Run(t, new(CacheStatusTestSuite))
 	suite.Run(t, new(RuleMatcherTestSuite))
+	suite.Run(t, new(EntryTestSuite))
 }

caddyfile.go

@@ -43,7 +43,7 @@ var (
 	defaultcacheBucketsNum        = 256
 	defaultCacheMaxMemorySize     = GB // default is 1 GB
 	defaultRedisConnectionSetting = "localhost:6379 0"
-	defaultCacheKeyTemplate       = "{http.request.method} {http.request.host}{http.request.uri.path} {http.request.uri.query}"
+	defaultCacheKeyTemplate       = "{http.request.method} {http.request.host}{http.request.uri.path}?{http.request.uri.query}"
 	// Note: prevent character space in the key
 	// the key is refereced from github.com/caddyserver/caddy/v2/modules/caddyhttp.addHTTPVarsToReplacer
 )
@@ -63,7 +63,9 @@ const (
 	// format: addr db password or addr db or addr
 	// ex.
 	// localhost:6789 0 => connect without password. only index and host:port provided
+	// the following are keys for extensions
 	keyDistributed = "distributed"
+	keyInfluxLog   = "influxlog"
 )
 
 func init() {
@@ -190,23 +192,23 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				}
 
 				cacheRule := &HeaderRuleMatcher{Header: args[0], Value: args[1:]}
-				d, _ := json.Marshal(cacheRule)
+				data, _ := json.Marshal(cacheRule)
 
 				config.RuleMatchersRaws = append(config.RuleMatchersRaws, RuleMatcherRawWithType{
 					Type: MatcherTypeHeader,
-					Data: d,
+					Data: data,
 				})
 
 			case keyMatchPath:
 				if len(args) != 1 {
 					return d.Err("Invalid usage of match_path in cache config.")
 				}
 				cacheRule := &PathRuleMatcher{Path: args[0]}
-				d, _ := json.Marshal(cacheRule)
+				data, _ := json.Marshal(cacheRule)
 
 				config.RuleMatchersRaws = append(config.RuleMatchersRaws, RuleMatcherRawWithType{
 					Type: MatcherTypePath,
-					Data: d,
+					Data: data,
 				})
 
 			case keyCacheKey:
@@ -238,7 +240,7 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 
 				unm, ok := mod.New().(caddyfile.Unmarshaler)
 				if !ok {
-					return d.Errf("distributed module '%s' is not a Caddfile unmarshaler", mod)
+					return d.Errf("distributed module '%s' is not a Caddyfile unmarshaler", mod)
 				}
 
 				err = unm.UnmarshalCaddyfile(d.NewFromNextSegment())
@@ -248,6 +250,13 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 
 				h.DistributedRaw = caddyconfig.JSONModuleObject(unm, "distributed", "consul", nil)
 
+			// case keyInfluxLog:
+			// 	raw, err := setupInfluxLog(keyInfluxLog, d, args)
+			// 	if err != nil {
+			// 		return err
+			// 	}
+			// 	h.InfluxLogRaw = raw
+
 			default:
 				return d.Err("Unknown cache parameter: " + parameter)
 			}
@@ -259,6 +268,28 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	return nil
 }
 
+func setupInfluxLog(key string, d *caddyfile.Dispenser, args []string) (json.RawMessage, error) {
+
+	mod, err := caddy.GetModule("caddy.logging.writers." + key)
+	if err != nil {
+		return nil, d.Errf("getting influxlog module '%s': '%v'", mod, err)
+	}
+
+	unm, ok := mod.New().(caddyfile.Unmarshaler)
+	if !ok {
+		return nil, d.Errf("influxlog module '%s' is not a Caddyfile unmarshaler", mod)
+	}
+
+	err = unm.UnmarshalCaddyfile(d.NewFromNextSegment())
+	if err != nil {
+		return nil, err
+	}
+
+	raw := caddyconfig.JSONModuleObject(unm, "mylog", "influxlog", nil)
+
+	return raw, nil
+}
+
 // Interface guards
 var (
 	_ caddyfile.Unmarshaler = (*Handler)(nil)

caddyfile_test.go

@@ -63,6 +63,19 @@ func (suite *CaddyfileTestSuite) TestInvalidParameter() {
 	suite.Error(err, "invalid parameter")
 }
 
+func (suite *CaddyfileTestSuite) TestRedisConnectionSetting() {
+	h := httpcaddyfile.Helper{
+		Dispenser: caddyfile.NewTestDispenser(`
+        http_cache {
+            redis_connection_setting localhost:6379 2 pass 5
+        }
+        `),
+	}
+
+	_, err := parseCaddyfile(h)
+	suite.Error(err, "invalid usage of redis_connection_setting in cache config.")
+}
+
 func (suite *CaddyfileTestSuite) TestDistributedCacheConfig() {
 	h := httpcaddyfile.Helper{
 		Dispenser: caddyfile.NewTestDispenser(`

cmd/main.go

@@ -31,6 +31,7 @@ import (
 
 	// plug in Caddy modules here
 	_ "github.com/sillygod/cdp-cache"
+	_ "github.com/sillygod/cdp-cache/extends/influxlog"
 
 	_ "github.com/caddyserver/caddy/v2/modules/standard"
 )

example/influxdb_monitor/Caddyfile

@@ -0,0 +1,41 @@
+{
+    order http_cache before reverse_proxy
+    admin 0.0.0.0:7777
+    debug
+}
+
+:9991 {
+    reverse_proxy {
+        to localhost:9995
+    }
+
+    http_cache {
+        cache_type in_memory 
+        match_path /
+
+    }
+
+    log {
+        output influxlog {
+            addr http://localhost:8086
+            token "user:pass1234"
+            organization ""
+            bucket telegraf
+            # For influxDB 1.8 the token should follow the rule below
+            # https://github.com/influxdata/influxdb-client-go#influxdb-18-api-compatibility
+        }
+        level debug
+    }
+
+
+}
+
+:9995 {
+    header Cache-control "public"
+    root * /tmp/caddy-benchmark
+    file_server
+
+    log {
+        level info
+    }
+}

example/influxdb_monitor/docker-compose.yml

@@ -0,0 +1,88 @@
+version: "3.7"
+
+services:
+  influxdb:
+    image: influxdb:1.8.1
+    volumes:
+      - influxdb:/var/lib/influxdb
+      - ./example/influxdb_monitor/influxdb.conf:/etc/influxdb/influxdb.conf
+    environment:
+      - INFLUXDB_ADMIN_USER=admin
+      - INFLUXDB_ADMIN_PASSWORD=pass1234
+      - INFLUXDB_USER=user
+      - INFLUXDB_USER_PASSWORD=pass1234
+
+    restart: always
+    networks:
+      - influxdata
+
+
+    ports:
+      - "8086:8086"
+      - "8082:8082"
+
+  # telegraf to collects the syslog 
+  telegraf:
+    image: telegraf:1.15.1
+    environment:
+      - HOSTNAME=songa-telegraf
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./example/influxdb_monitor/telegraf.conf:/etc/telegraf/telegraf.conf
+
+    restart: always
+
+    networks:
+      - influxdata
+
+    depends_on:
+      - influxdb
+
+  chronograf:
+    image: chronograf:1.8.4
+    environment:
+      - RESOURCES_PATH=/usr/share/chronograf/resources
+    volumes:
+      - chronograf:/var/lib/chronograf
+    ports:
+      - "8888:8888"
+
+    restart: always
+
+    networks:
+      - influxdata
+
+    depends_on:
+      - influxdb
+      - telegraf
+
+  # in the future add the grafana log panel
+
+  # cdp to write the log as syslog
+  cdp: &cdp-source
+    image: golang:1.14.2
+    container_name: cdp
+    volumes:
+      - cdp:/go/
+      - go-build-cache:/root/.cache/go-build
+      - .:${PROJECT_PATH}
+
+    command: >
+      sh -c "echo 'spin up caddy proxy cache...' &&
+      cd ${PROJECT_PATH} &&
+      go run $PROJECT_PATH/cmd/main.go run --config example/influxdb_monitor/Caddyfile"
+
+    ports:
+      - "9991:9991"
+    environment:
+      - PROJECT_PATH=${PROJECT_PATH}
+
+
+volumes:
+  cdp:
+  go-build-cache:
+  influxdb:
+  chronograf:
+
+networks:
+  influxdata: