Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions configurationtypes/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,26 @@ type CacheProvider struct {
Configuration interface{} `json:"configuration" yaml:"configuration"`
}

func (c *CacheProvider) MarshalJSON() ([]byte, error) {
if !c.Found && c.URL == "" && c.Path == "" && c.Configuration == nil && c.Uuid == "" {
return []byte("null"), nil
}

return json.Marshal(struct {
Uuid string
Found bool `json:"found"`
URL string `json:"url"`
Path string `json:"path"`
Configuration interface{} `json:"configuration"`
}{
Uuid: c.Uuid,
Found: c.Found,
URL: c.URL,
Path: c.Path,
Configuration: c.Configuration,
})
}

// Timeout configuration to handle the cache provider and the
// reverse-proxy timeout.
type Timeout struct {
Expand Down
71 changes: 41 additions & 30 deletions docs/e2e/Souin E2E.postman_collection.json

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions docs/website/content/docs/middlewares/caddy.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ route {
cache
}
```
With this given configuration if you go on [https://localhost/souin-api/souin](https://localhost/souin-api/souin) we get the stored keys list.
If we go on [https://localhost/souin-api/metrics](https://localhost/souin-api/metrics) we access to the prometheus web page.
If we go on [https://localhost/souin-api/debug/](https://localhost/souin-api/debug/) we access to the pprof web page.
With this given configuration if you go on [https://localhost:2019/souin-api/souin](https://localhost:2019/souin-api/souin) we get the stored keys list.
If we go on [https://localhost:2019/souin-api/metrics](https://localhost/souin-api/metrics) we access to the prometheus web page.
If we go on [https://localhost:2019/souin-api/debug/](https://localhost/souin-api/debug/) we access to the pprof web page.

### Complex configuration

Expand Down
2 changes: 1 addition & 1 deletion docs/website/content/docs/uses-cases/api-platform.md
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ api_platform:

+ http_cache:
+ invalidation:
+ urls: [ 'http://php/souin-api/souin' ]
+ urls: [ 'http://php:2019/souin-api/souin' ]
+ purger: api_platform.http_cache.purger.souin
```

Expand Down
69 changes: 43 additions & 26 deletions pkg/api/souin.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ func (s *SouinAPI) BulkDelete(key string, purge bool) {
if purge {
current.Delete(core.MappingKeyPrefix + key)
}

current.Delete(key)
}

s.Delete(key)
Expand Down Expand Up @@ -162,38 +164,48 @@ var storageToInfiniteTTLMap = map[string]time.Duration{
types.DefaultStorageName: types.OneYearDuration,
}

func (s *SouinAPI) purgeMapping() {
func EvictMapping(current types.Storer) {
values := current.MapKeys(core.MappingKeyPrefix)
now := time.Now()
for _, current := range s.storers {
infiniteStoreDuration := storageToInfiniteTTLMap[current.Name()]
values := current.MapKeys(core.MappingKeyPrefix)
for k, v := range values {
mapping := &core.StorageMapper{}
infiniteStoreDuration := storageToInfiniteTTLMap[current.Name()]

e := proto.Unmarshal([]byte(v), mapping)
if e != nil {
current.Delete(core.MappingKeyPrefix + k)
continue
}
for k, v := range values {
mapping := &core.StorageMapper{}

updated := false
for key, val := range mapping.GetMapping() {
if now.Sub(val.FreshTime.AsTime()) > 0 && now.Sub(val.StaleTime.AsTime()) > 0 {
delete(mapping.GetMapping(), key)
updated = true
}
e := proto.Unmarshal([]byte(v), mapping)
if e != nil {
current.Delete(core.MappingKeyPrefix + k)
continue
}

updated := false
for key, val := range mapping.GetMapping() {
if now.Sub(val.FreshTime.AsTime()) > 0 && now.Sub(val.StaleTime.AsTime()) > 0 {
delete(mapping.GetMapping(), key)
updated = true
}
}

if updated {
v, e := proto.Marshal(mapping)
if e != nil {
fmt.Println("Impossible to re-encode the mapping", core.MappingKeyPrefix+k)
current.Delete(core.MappingKeyPrefix + k)
}
_ = current.Set(core.MappingKeyPrefix+k, v, infiniteStoreDuration)
if updated {
v, e := proto.Marshal(mapping)
if e != nil {
fmt.Println("Impossible to re-encode the mapping", core.MappingKeyPrefix+k)
current.Delete(core.MappingKeyPrefix + k)
}
_ = current.Set(core.MappingKeyPrefix+k, v, infiniteStoreDuration)
}

if len(mapping.GetMapping()) == 0 {
current.Delete(core.MappingKeyPrefix + k)
}
}
time.Sleep(time.Minute)
}

func (s *SouinAPI) purgeMapping() {
for _, current := range s.storers {
EvictMapping(current)
}

fmt.Println("Successfully clear the mappings.")
}
Expand Down Expand Up @@ -228,7 +240,9 @@ func (s *SouinAPI) HandleRequest(w http.ResponseWriter, r *http.Request) {
keysToInvalidate := []string{}
switch invalidator.Type {
case groupInvalidationType:
keysToInvalidate, _ = s.surrogateStorage.Purge(http.Header{"Surrogate-Key": invalidator.Groups})
var surrogateKeys []string
keysToInvalidate, surrogateKeys = s.surrogateStorage.Purge(http.Header{"Surrogate-Key": invalidator.Groups})
keysToInvalidate = append(keysToInvalidate, surrogateKeys...)
case uriPrefixInvalidationType, uriInvalidationType:
bodyKeys := []string{}
listedKeys := s.GetAll()
Expand Down Expand Up @@ -311,10 +325,13 @@ func (s *SouinAPI) HandleRequest(w http.ResponseWriter, r *http.Request) {
}
}
} else {
ck, _ := s.surrogateStorage.Purge(r.Header)
ck, surrogateKeys := s.surrogateStorage.Purge(r.Header)
for _, k := range ck {
s.BulkDelete(k, true)
}
for _, k := range surrogateKeys {
s.BulkDelete("SURROGATE_"+k, true)
}
}
w.WriteHeader(http.StatusNoContent)
default:
Expand Down
5 changes: 2 additions & 3 deletions pkg/middleware/middleware.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ func registerMappingKeysEviction(logger core.Logger, storers []types.Storer) {
go func(current types.Storer) {
for {
logger.Debugf("run mapping eviction for storer %s", current.Name())
current.MapKeys(core.MappingKeyPrefix)
time.Sleep(time.Minute)

api.EvictMapping(current)
}
}(storer)
}
Expand Down Expand Up @@ -731,7 +731,6 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
}
for _, currentStorer := range s.Storers {
fresh, stale = currentStorer.GetMultiLevel(finalKey, req, validator)
fmt.Printf("modecontext: %#v\n%#v\n%#v\n\n", modeContext, fresh, stale)

if fresh != nil || stale != nil {
storerName = currentStorer.Name()
Expand Down
10 changes: 9 additions & 1 deletion pkg/storage/defaultProvider.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ type Default struct {
m *sync.Map
stale time.Duration
logger core.Logger

mu sync.Mutex
}

type item struct {
Expand All @@ -43,6 +45,9 @@ func (provider *Default) Uuid() string {

// MapKeys method returns a map with the key and value
func (provider *Default) MapKeys(prefix string) map[string]string {
provider.mu.Lock()
defer provider.mu.Unlock()

now := time.Now()
keys := map[string]string{}

Expand Down Expand Up @@ -200,6 +205,9 @@ func (provider *Default) Init() error {

// Reset method will reset or close provider
func (provider *Default) Reset() error {
provider.m = &sync.Map{}
provider.mu.Lock()
provider.m = new(sync.Map)
provider.mu.Unlock()

return nil
}
5 changes: 4 additions & 1 deletion pkg/surrogate/providers/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func (s *baseStorage) purgeTag(tag string) []string {
}

// Store will take the lead to store the cache key for each provided Surrogate-key
func (s *baseStorage) Store(response *http.Response, cacheKey, _ string) error {
func (s *baseStorage) Store(response *http.Response, cacheKey, uri string) error {
h := response.Header

cacheKey = url.QueryEscape(cacheKey)
Expand All @@ -230,18 +230,21 @@ func (s *baseStorage) Store(response *http.Response, cacheKey, _ string) error {
if controls := s.ParseHeaders(v); len(controls) != 0 {
if len(controls) == 1 && controls[0] == "" {
s.storeTag(key, cacheKey, urlRegexp)
s.storeTag(uri, cacheKey, urlRegexp)

continue
}
for _, control := range controls {
if s.parent.candidateStore(control) {
s.storeTag(key, cacheKey, urlRegexp)
s.storeTag(uri, cacheKey, urlRegexp)

break
}
}
} else {
s.storeTag(key, cacheKey, urlRegexp)
s.storeTag(uri, cacheKey, urlRegexp)
}
}

Expand Down
16 changes: 6 additions & 10 deletions plugins/caddy/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package httpcache

import (
"fmt"
"github.com/caddyserver/caddy/v2"
"github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/api"
"net/http"
"strings"
"time"

"github.com/caddyserver/caddy/v2"
"github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/api"

"github.com/darkweak/storages/core"
)

Expand Down Expand Up @@ -73,16 +74,11 @@ func (a *adminAPI) Provision(ctx caddy.Context) error {
return nil
}

// Routes returns the admin routes for the PKI app.
// Routes returns the admin routes.
func (a *adminAPI) Routes() []caddy.AdminRoute {
basepath := "/souin-api"
if a.app != nil && a.app.API.BasePath != "" {
basepath = a.app.API.BasePath
}

return []caddy.AdminRoute{
{
Pattern: basepath + "/{params...}",
Pattern: "/{params...}",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will likely cause issues with other modules that add to the admin API. It's not namespaced, so the souin API endpoint ends up being a catch-all for all requests. Running this config:

{
	cache {
		api {
			debug
			prometheus
			souin
		}
	}
	debug
}
localhost {
	route {
		cache
		reverse_proxy localhost:8080
	}
}

http://localhost:8080 {
	respond "hi"
}

I get this response for API requests unrelated to souin:

$ curl http://localhost:2019/test/
{"error":"resource not found: /test/"}

Which is coming from here:

return caddy.APIError{
HTTPStatus: http.StatusNotFound,
Err: fmt.Errorf("resource not found: %v", request.URL.Path),
}
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@darkweak, I think this is critical because it may be an unintended side-effect

Handler: caddy.AdminHandlerFunc(a.handleAPIEndpoints),
},
}
Expand Down
2 changes: 0 additions & 2 deletions plugins/caddy/httpcache.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,6 @@ func (s *SouinCaddyMiddleware) FromApp(app *SouinApp) error {
}
}

s.Configuration.API = app.API

if s.Configuration.GetDefaultCache() == nil {
s.Configuration.DefaultCache = DefaultCache{
AllowedHTTPVerbs: app.DefaultCache.AllowedHTTPVerbs,
Expand Down
Loading
Loading