Finalized ru cache for metadata

Author: sadoyan

README.md

@@ -23,13 +23,11 @@ Building from a source is also easy.
 git clone  https://github.com/sadoyan/go-webrados.git
 cd go-webrados
 export GOROOT=/path/to/your/go
-./build.sh
-
-./webrados /path/to/config.ini
+go mod tidy
+do build .
 ```
 
 
-
 ### **Configuration**
 ---------
 
@@ -42,7 +40,6 @@ listen : 0.0.0.0:8080
 dispatchers : 20
 serveruser : admin
 serverpass : 261a5983599fd57a016122ec85599ec4
-uploadmaxpart : 52428800
 dangerzone : yes
 readonly : no
 authread : no
@@ -52,33 +49,35 @@ logfile : no
 logpath: /opt/webrados.log
 allpools: no
 poollist: bublics, donuts, images
-database : redis
 
-[redis]
+[cache]
+shards: 1024
+lifewindow: 10
+cleanwindow: 1
+maxrntriesinwindow: 600000
+maxentrysize: 5000
+maxcachemb: 1024
+
+[database]
+type : ceph
 server: 127.0.0.1:6379
 username : none
 password : none
 database : 0
 
-[mysql]
-username : ceph
-password : ceph
-dbname : ceph
-server : 127.0.0.1:3306
-
 [monitoring]
 enabled : true
 url:  127.0.0.1:9090
 user: admin
 pass: admin
 ```
+
 ### **API**
 ---------
 
 | **Name** | **Description** |
 | ------------- | ------------- |
 |**Read File**|HTTP ```GET:``` http://{BINDADDRESS}/{POOLNAME}/{FILENAME}|
-|**Read File**|HTTP ```GET:``` http://{BINDADDRESS}/{POOLNAME}/{FILENAME}|
 |**Upload File**|HTTP ```POST, PUT:``` http://{BINDADDRESS}/{POOLNAME}/{FILENAME}|
 |**Remove File**|HTTP ```DELETE:``` http://{BINDADDRESS}/{POOLNAME}/{FILENAME}|
 
@@ -87,23 +86,36 @@ Configuration file is pretty simple and intuitive.
 ### **Section main**
 ---------
 
-| **Name**  | **Description** |
-| ------------- | ------------- |
-|**listen**|IP port to bind.|
-|**dispatchers**|Number of threads for webserver.|
-|**serveruser**|Static user.|
-|**serverpass**|MD5 hash of password for static user. It can be the output of `echo -n SecretPaSs | md5sum |awk '{print $1}'`|
-|**dangerzone**|Enable destructive methods and commands (DELETE).|
-|**readonly**|Enable readonly mode. If 'yes' only GET is allowed.|
-|**authread**|Require authentication for GET only.|
-|**authwrite**|Require authentication for POST/PUT/DELETE.|
-|**radoconns**|Number of connection to CEPH.|
-|**logfile**|Log to file, if 'no' logs are sent to stdout.|
-|**logpath**|Path for log file.|
-|**uploadmaxpart**| Maximum file chunk size (Sbould be amaller or erqual ro `osd max object size`).| 
-|**allpools:**|yes/no . If yes program will scan ceph and enable access via web to all pool.| 
-|**poollist:**|Works only if **allpools** is set to **no**. Coma separated list of pools which should be accesible via webrados program.|  
-|**redis**|IP address and port of Redis server.|
+| **Name**          | **Description**                                                                                                                      |
+|-------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+| **listen**        | IP port to bind.                                                                                                                     |
+| **dispatchers**   | Number of threads for webserver.                                                                                                     |
+| **serveruser**    | Static user.                                                                                                                         |
+| **serverpass**    | MD5 hash of password for static user. It can be the output of `echo -n SecretPaSs md5sum |awk '{print $1}'`|                                                   | 
+| **dangerzone**    | Enable destructive methods and commands (DELETE).                                                                                    |
+| **readonly**      | Enable readonly mode. If 'yes' only GET is allowed.                                                                                  |
+| **authread**      | Require authentication for GET only.                                                                                                 |
+| **authwrite**     | Require authentication for POST/PUT/DELETE.                                                                                          |
+| **radoconns**     | Number of connection to CEPH.                                                                                                        |
+| **logfile**       | Log to file, if 'no' logs are sent to stdout.                                                                                        |
+| **logpath**       | Path for log file.                                                                                                                   |
+| **uploadmaxpart** | Maximum file chunk size (Sbould be amaller or erqual ro `osd max object size`).                                                      | 
+| **allpools:**     | yes/no . If yes program will scan ceph and enable access via web to all pool.                                                        | 
+| **poollist:**     | Works only if **allpools** is set to **no**. Coma separated list of pools which should be accesible via webrados program.            |  
+| **database**      | Name of database server for storing metadata of fliles (Ceph, Redis, MySQL). MySQL and Redis are deprecated and will be removed soon |
+
+### **Section cache**
+---------
+| **Name**  | **Description**                                                                                   |
+| ------------- |---------------------------------------------------------------------------------------------------|
+|**shards**| Number of shards (must be a power of 2)                                                           |
+|**lifewindow**| Time after which entry can be evicted                                                             |
+|**cleanwindow**| Interval between removing expired entries (clean up). If set to <= 0 then no action is performed. |
+|**maxrntriesinwindow**| rps * lifeWindow, used only in initial memory allocation                                          |
+|**maxentrysize**| max entry size in bytes, used only in initial memory allocation                                   |
+|**maxcachemb**| Cache will not allocate more memory than this limit, value in MB.  0 value means no size limit                                |
+
+
 ### **Section monitoring**
 ---------
 | **Name**  | **Description** |
@@ -123,9 +135,7 @@ GO-Webrados will periodically read ```uesrs.txt``` file and automatically update
 ### **Large files**
 
 In order to be able to store large file in RADOS directly files needs to be split to smaller chunks. 
-GO-WebRados will automatically set maximum chunk size to  **uploadmaxpart** and split files in accordance to that. 
-For this GO-WebRados requires Redis servers to store metadata for big files. (Make sure to keep Redis database safe and periodically back it up.)
-Redis will store metadata only for large files. Small files are to split and not require special metadata server.  
+GO-WebRados will automatically set maximum chunk size to  **OSDMaxObjectSize** of Ceph and split files in accordance to that. 
 
 ### **Special commands**
 

build.sh

@@ -1,16 +1,5 @@
 #!/bin/bash
 
 export GOROOT=/usr/local/go
-export GOPATH=`pwd`
-
-rm -rf pkg/*
-rm -rf src/{github.com,golang.org,gopkg.in}
-go get github.com/ceph/go-ceph
-go get gopkg.in/ini.v1
-go get golang.org/x/sys/unix
-go get go.opentelemetry.io/otel
-go get github.com/gomodule/redigo/redis
-go get github.com/go-sql-driver/mysql
-#go get github.com/go-redis/redis
-
-#go build src/webrados.go
+go mod tidy
+go build webrados.go

config.ini

@@ -12,9 +12,15 @@ logfile : no
 logpath: /opt/webrados.log
 allpools: no
 poollist: bublics, donuts, images
-database : ceph
-cacheitems: 6000000
-cachettl : 300
+
+[cache]
+shards: 1024
+lifewindow: 10
+cleanwindow: 1
+maxrntriesinwindow: 600000
+maxentrysize: 5000
+maxcachemb: 1024
+
 
 [database]
 type : ceph

configs/cfg.go

@@ -5,11 +5,9 @@ import (
 	"github.com/ceph/go-ceph/rados"
 	"gopkg.in/ini.v1"
 	"log"
-	"math"
 	"os"
 	"strconv"
 	"strings"
-	"time"
 )
 
 type CfgType struct {
@@ -47,8 +45,14 @@ type CfgType struct {
 	MySQLUser        string
 	MySQLPassword    string
 	DBType           string
-	CacheItems       int
-	CacheTTL         time.Duration
+	//CacheItems         int
+	//CacheTTL           time.Duration
+	CacheShards             int
+	CacheLifeWindow         int
+	CacheCleanWindow        int
+	CacheMaxEntriesInWindow int
+	CacheMaxEntrySize       int
+	CacheHardMaxCacheSize   int
 }
 
 var Conf = &CfgType{
@@ -84,9 +88,15 @@ var Conf = &CfgType{
 	MySQLDB:          "",
 	MySQLUser:        "",
 	MySQLPassword:    "",
-	DBType:           "",
-	CacheItems:       0,
-	CacheTTL:         math.MaxInt,
+	DBType:           "ceph",
+	//CacheItems:       0,
+	//CacheTTL:         math.MaxInt,
+	CacheShards:             1024,
+	CacheLifeWindow:         1024,
+	CacheCleanWindow:        20,
+	CacheMaxEntriesInWindow: 600000,
+	CacheMaxEntrySize:       5000,
+	CacheHardMaxCacheSize:   1024,
 }
 
 var authorized = make(map[string]string, 10)
@@ -103,6 +113,14 @@ func stringTObool(key string, value string) bool {
 	return false
 }
 
+func stringTOint(key string) int {
+	a, err := strconv.Atoi(key)
+	if err != nil {
+		log.Fatal("\n Value for " + key + " should be  numeric\n")
+	}
+	return a
+}
+
 var Cfgfile = "config.ini"
 
 func SetVarsik() {
@@ -120,26 +138,18 @@ func SetVarsik() {
 	}
 
 	Conf.HttpAddress = cfg.Section("main").Key("listen").String()
-	Conf.DispatchersCount, _ = cfg.Section("main").Key("dispatchers").Int()
+	Conf.DispatchersCount = stringTOint(cfg.Section("main").Key("dispatchers").String())
 	Conf.InternalQueue, _ = cfg.Section("main").Key("internalqueue").Bool()
 	qs, _ := cfg.Section("main").Key("queuesize").Int()
 	Conf.queue = make(chan string, qs)
-
-	//Conf.Uploadmaxpart, err = cfg.Section("main").Key("uploadmaxpart").Int()
-	//if err != nil {
-	//	log.Fatal("Please set numeric value to Uploadmaxpart")
-	//}
 	vsyo, _ := rados.NewConn()
 	_ = vsyo.ReadDefaultConfigFile()
 	_ = vsyo.Connect()
 	osdMaxObjectSize, _ := vsyo.GetConfigOption("osd max object size")
 	s, _ := strconv.Atoi(osdMaxObjectSize)
 	Conf.Uploadmaxpart = s
 
-	Conf.Radoconns, err = cfg.Section("main").Key("radoconns").Int()
-	if err != nil {
-		log.Fatal("Please set numeric value to Radoconns")
-	}
+	Conf.Radoconns = stringTOint(cfg.Section("main").Key("radoconns").String())
 
 	Conf.LogStdout = stringTObool("dangerzone", strings.ToLower(cfg.Section("main").Key("logfile").String()))
 	Conf.Logfile = cfg.Section("main").Key("logpath").String()
@@ -181,19 +191,14 @@ func SetVarsik() {
 		Conf.DangeZone = stringTObool("dangerzone", strings.ToLower(cfg.Section("main").Key("dangerzone").String()))
 	}
 
+	Conf.CacheShards = stringTOint(cfg.Section("cache").Key("shards").String())
+	Conf.CacheLifeWindow = stringTOint(cfg.Section("cache").Key("lifewindow").String())
+	Conf.CacheCleanWindow = stringTOint(cfg.Section("cache").Key("cleanwindow").String())
+	Conf.CacheMaxEntriesInWindow = stringTOint(cfg.Section("cache").Key("maxrntriesinwindow").String())
+	Conf.CacheMaxEntrySize = stringTOint(cfg.Section("cache").Key("maxentrysize").String())
+	Conf.CacheHardMaxCacheSize = stringTOint(cfg.Section("cache").Key("maxcachemb").String())
+
 	Conf.DBType = cfg.Section("database").Key("type").String()
-	cache, cerr := cfg.Section("main").Key("cacheitems").Int()
-	if cerr != nil {
-		log.Fatal("Cache capacity should be numeric", cerr)
-	} else {
-		Conf.CacheItems = cache
-	}
-	cachettl, cttlerr := cfg.Section("main").Key("cachettl").Int()
-	if cttlerr != nil {
-		log.Fatal("Cache TTL should be numeric", cttlerr)
-	} else {
-		Conf.CacheTTL = time.Duration(cachettl)
-	}
 	switch Conf.DBType {
 	case "redis":
 		Conf.RedisServer = cfg.Section("database").Key("server").String()
@@ -218,6 +223,7 @@ func SetVarsik() {
 		Conf.MySQLUser = cfg.Section("database").Key("username").String()
 		Conf.MySQLPassword = cfg.Section("database").Key("password").String()
 	case "ceph":
+		Conf.DBType = "ceph"
 	}
 
 }

deplohu.sh

@@ -10,6 +10,7 @@ require (
 )
 
 require (
+	github.com/allegro/bigcache/v3 v3.1.0 // indirect
 	github.com/ceph/go-ceph v0.19.1-0.20230112054424-122159ed21a1 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/gomodule/redigo v1.8.9 // indirect

go.mod

@@ -1,3 +1,5 @@
+github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk=
+github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I=
 github.com/aws/aws-sdk-go v1.44.151/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go v1.44.172/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/ceph/go-ceph v0.19.0/go.mod h1:sdTcqdDeIPWX3TaR5HCi5YtT+BliI6fFvvWP6Io7VQE=

go.sum

@@ -19,43 +19,25 @@ var myConns = 5
 //var Cache LRUCache
 
 var config = bigcache.Config{
-	// number of shards (must be a power of 2)
-	Shards: 1024,
-	// time after which entry can be evicted
-	LifeWindow: 5 * time.Minute,
-	// Interval between removing expired entries (clean up).
-	// If set to <= 0 then no action is performed.
-	// Setting to < 1 second is counterproductive — bigcache has a one second resolution.
-	CleanWindow: 1 * time.Minute,
-	// rps * lifeWindow, used only in initial memory allocation
-	MaxEntriesInWindow: 1000 * 10 * 60,
-	// max entry size in bytes, used only in initial memory allocation
-	MaxEntrySize: 5000,
-	// prints information about additional memory allocation
-	Verbose: true,
-	// cache will not allocate more memory than this limit, value in MB
-	// if value is reached then the oldest entries can be overridden for the new ones
-	// 0 value means no size limit
-	HardMaxCacheSize: 1024,
-	// callback fired when the oldest entry is removed because of its expiration time or no space left
-	// for the new entry, or because delete was called. A bitmask representing the reason will be returned.
-	// Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
-	OnRemove: nil,
-	// OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left
-	// for the new entry, or because delete was called. A constant representing the reason will be passed through.
-	// Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
-	// Ignored if OnRemove is specified.
-	OnRemoveWithReason: nil,
+	Shards:             configs.Conf.CacheShards,
+	LifeWindow:         time.Duration(configs.Conf.CacheLifeWindow) * time.Minute,
+	CleanWindow:        time.Duration(configs.Conf.CacheCleanWindow) * time.Minute,
+	MaxEntriesInWindow: configs.Conf.CacheMaxEntriesInWindow,
+	MaxEntrySize:       configs.Conf.CacheMaxEntrySize,
+	Verbose:            true,
+	HardMaxCacheSize:   configs.Conf.CacheHardMaxCacheSize,
+	OnRemove:           nil, // callback on remove
+	OnRemoveWithReason: nil, // callback on remove with reason
 }
 var Cache, _ = bigcache.New(context.Background(), config)
 
 func DBConnect() {
-	go func() {
-		for {
-			log.Println("[Cache statistics. Len:", Cache.Len(), "Cap:", Cache.Capacity(), "Hits:", Cache.Stats().Hits, "Misses:", strconv.Itoa(int(Cache.Stats().Misses))+"]")
-			time.Sleep(5 * time.Second)
-		}
-	}()
+	//go func() {
+	//	for {
+	//		log.Println("[Cache statistics. Len:", Cache.Len(), "Cap:", Cache.Capacity(), "Hits:", Cache.Stats().Hits, "Misses:", strconv.Itoa(int(Cache.Stats().Misses))+"]")
+	//		time.Sleep(5 * time.Second)
+	//	}
+	//}()
 	switch configs.Conf.DBType {
 	case "ceph":
 		log.Println("[Using Rados file xattrs for metadata]")
@@ -103,9 +85,9 @@ func DBClient(filename string, ops string, id string) (string, error) {
 				file, err = cephget(filename)
 				if err == nil {
 
-					//for i := 1; i <= 100000; i++ {
-					//	_ = Cache.Set(filename+strconv.Itoa(i), []byte(file))
-					//}
+					for i := 1; i <= 100000; i++ {
+						_ = Cache.Set(filename+strconv.Itoa(i), []byte(file))
+					}
 
 					_ = Cache.Set(filename, []byte(file))
 					//stattemp = "Fresh:"