Connect to a multi-nodal distributed database system

.github/workflows/test.yml

@@ -14,5 +14,13 @@ jobs:
     - run: sudo add-apt-repository ppa:inkscape.dev/stable
     - run: sudo apt-get update
     - run: sudo apt-get install inkscape -y
+    - name: Set up MariaDB
+      id: mariadb
+      uses: rusher/action-setup-mariadb@v1.6
+      with:
+        tag: '11.4'
+        local: true  # Force local installation
+        root-password: 'myRootPassword'
+        database: 'myDb'
     - run: go build ./cmd/violet/
-    - run: go test ./...
+    - run: DB="root:myRootPassword@tcp(127.0.0.1)/myDb" go test ./...

Makefile

@@ -0,0 +1,6 @@
+.PHONY: all sqlc
+
+all: sqlc
+
+sqlc:
+	sqlc generate

certs/certs.go

@@ -7,7 +7,6 @@ import (
 	"github.com/1f349/violet/logger"
 	"github.com/1f349/violet/utils"
 	"github.com/mrmelon54/certgen"
-	"github.com/mrmelon54/rescheduler"
 	"io/fs"
 	"math/big"
 	"os"
@@ -28,13 +27,12 @@ type Certs struct {
 	m    map[string]*tls.Certificate
 	ca   *certgen.CertGen
 	sn   atomic.Int64
-	r    *rescheduler.Rescheduler
 	t    *time.Ticker
 	ts   chan struct{}
 }
 
 // New creates a new cert list
-func New(certDir fs.FS, keyDir fs.FS, selfCert bool) *Certs {
+func New(certDir fs.FS, keyDir fs.FS, selfCert bool, gap time.Duration) *Certs {
 	c := &Certs{
 		cDir: certDir,
 		kDir: keyDir,
@@ -45,15 +43,13 @@ func New(certDir fs.FS, keyDir fs.FS, selfCert bool) *Certs {
 	}
 
 	if !selfCert {
-		// the rescheduler isn't even used in self cert mode so why initialise it
-		c.r = rescheduler.NewRescheduler(c.threadCompile)
-
-		c.t = time.NewTicker(2 * time.Hour)
+		// the refresh loop isn't even used in self cert mode so why initialise it
+		c.t = time.NewTicker(gap)
 		go func() {
 			for {
 				select {
 				case <-c.t.C:
-					c.Compile()
+					c.threadCompile()
 				case <-c.ts:
 					return
 				}
@@ -121,18 +117,6 @@ func (c *Certs) GetCertForDomain(domain string) *tls.Certificate {
 	return nil
 }
 
-// Compile loads the certificates and keys from the directories.
-//
-// This method makes use of the rescheduler instead of just ignoring multiple
-// calls.
-func (c *Certs) Compile() {
-	// don't bother compiling in self-signed mode
-	if c.ss {
-		return
-	}
-	c.r.Run()
-}
-
 func (c *Certs) Stop() {
 	if c.t != nil {
 		c.t.Stop()

certs/certs_test.go

@@ -52,7 +52,7 @@ func TestCertsNew_Lookup(t *testing.T) {
 		},
 	}
 
-	certs := New(certDir, keyDir, false)
+	certs := New(certDir, keyDir, false, 5*time.Second)
 	assert.NoError(t, certs.internalCompile(certs.m))
 	cc := certs.GetCertForDomain("example.com")
 	leaf := certgen.TlsLeaf(cc)
@@ -67,7 +67,7 @@ func TestCertsNew_SelfSigned(t *testing.T) {
 		return
 	}
 
-	certs := New(nil, nil, true)
+	certs := New(nil, nil, true, 5*time.Second)
 	cc := certs.GetCertForDomain("example.com")
 	leaf := certgen.TlsLeaf(cc)
 	assert.Equal(t, []string{"example.com"}, leaf.DNSNames)

cmd/violet/conf.go

@@ -1,12 +1,16 @@
 package main
 
+import "github.com/1f349/violet/utils"
+
 type startUpConfig struct {
-	SelfSigned    bool         `json:"self_signed"`
-	ErrorPagePath string       `json:"error_page_path"`
-	Listen        listenConfig `json:"listen"`
-	InkscapeCmd   string       `json:"inkscape"`
-	RateLimit     uint64       `json:"rate_limit"`
-	MetricsToken  string       `json:"metrics_token"`
+	SelfSigned    bool               `json:"self_signed"`
+	ErrorPagePath string             `json:"error_page_path"`
+	Listen        listenConfig       `json:"listen"`
+	InkscapeCmd   string             `json:"inkscape"`
+	RateLimit     uint64             `json:"rate_limit"`
+	MetricsToken  string             `json:"metrics_token"`
+	TableRefresh  utils.DurationText `json:"table_refresh"`
+	CertRefresh   utils.DurationText `json:"cert_refresh"`
 }
 
 type listenConfig struct {

cmd/violet/serve.go

@@ -127,14 +127,20 @@ func (s *serveCmd) Execute(_ context.Context, _ *flag.FlagSet, _ ...interface{})
 	certDir := os.DirFS(filepath.Join(wd, "certs"))
 	keyDir := os.DirFS(filepath.Join(wd, "keys"))
 
+	dynamicErrorPages, err := errorPages.New(errorPageDir)
+	if err != nil {
+		logger.Logger.Fatal("Failed to load error pages", "err", err)
+	}
+
+	serviceCtx, cancelService := context.WithCancel(context.Background())
+
 	ws := websocket.NewServer()
-	allowedDomains := domains.New(db)                             // load allowed domains
-	acmeChallenges := utils.NewAcmeChallenge()                    // load acme challenge store
-	allowedCerts := certs.New(certDir, keyDir, config.SelfSigned) // load certificate manager
-	hybridTransport := proxy.NewHybridTransport(ws)               // load reverse proxy
-	dynamicFavicons := favicons.New(db, config.InkscapeCmd)       // load dynamic favicon provider
-	dynamicErrorPages := errorPages.New(errorPageDir)             // load dynamic error page provider
-	dynamicRouter := router.NewManager(db, hybridTransport)       // load dynamic router manager
+	allowedDomains := domains.New(serviceCtx, db, time.Duration(config.TableRefresh))                       // load allowed domains
+	acmeChallenges := utils.NewAcmeChallenge()                                                              // load acme challenge store
+	allowedCerts := certs.New(certDir, keyDir, config.SelfSigned, time.Duration(config.CertRefresh))        // load certificate manager
+	hybridTransport := proxy.NewHybridTransport(ws)                                                         // load reverse proxy
+	dynamicFavicons := favicons.New(db, config.InkscapeCmd)                                                 // load dynamic favicon provider
+	dynamicRouter := router.NewManager(serviceCtx, db, hybridTransport, time.Duration(config.TableRefresh)) // load dynamic router manager
 
 	// struct containing config for the http servers
 	srvConf := &conf.Conf{
@@ -150,7 +156,7 @@ func (s *serveCmd) Execute(_ context.Context, _ *flag.FlagSet, _ ...interface{})
 	}
 
 	// create the compilable list and run a first time compile
-	allCompilables := utils.MultiCompilable{allowedDomains, allowedCerts, dynamicFavicons, dynamicErrorPages, dynamicRouter}
+	allCompilables := utils.MultiCompilable{dynamicFavicons}
 	allCompilables.Compile()
 
 	_, httpsPort, ok := utils.SplitDomainPort(config.Listen.Https, 443)
@@ -219,6 +225,8 @@ func (s *serveCmd) Execute(_ context.Context, _ *flag.FlagSet, _ ...interface{})
 		os.Exit(1)
 	})
 
+	cancelService()
+
 	// stop updating certificates
 	allowedCerts.Stop()
 

cmd/violet/setup.go

@@ -21,6 +21,7 @@ import (
 	"path"
 	"path/filepath"
 	"strconv"
+	"time"
 )
 
 type setupCmd struct {
@@ -154,7 +155,7 @@ func (s *setupCmd) Execute(_ context.Context, _ *flag.FlagSet, _ ...interface{})
 
 	// domain manager to add a domain, no need to compile here as the program needs
 	// to be run again with the serve subcommand
-	allowedDomains := domains.New(db)
+	allowedDomains := domains.New(context.Background(), db, 5*time.Second)
 	allowedDomains.Put(answers.FirstDomain, true)
 
 	// don't bother with this part is the api won't be listening
@@ -181,7 +182,7 @@ func (s *setupCmd) Execute(_ context.Context, _ *flag.FlagSet, _ ...interface{})
 
 		// add with the route manager, no need to compile as this will run when opened
 		// with the serve subcommand
-		routeManager := router.NewManager(db, proxy.NewHybridTransportWithCalls(&nilTransport{}, &nilTransport{}, &websocket.Server{}))
+		routeManager := router.NewManager(context.Background(), db, proxy.NewHybridTransportWithCalls(&nilTransport{}, &nilTransport{}, &websocket.Server{}), 5*time.Minute)
 		err = routeManager.InsertRoute(target.RouteWithActive{
 			Route: target.Route{
 				Src:   path.Join(apiUrl.Host, apiUrl.Path),

database/migrations/20240308125121_domains.down.sql

@@ -0,0 +1 @@
+DROP TABLE domains;

database/migrations/20240308125121_domains.up.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS domains
+(
+    id     INTEGER PRIMARY KEY AUTO_INCREMENT,
+    domain TEXT UNIQUE NOT NULL,
+    active BOOLEAN     NOT NULL DEFAULT 1
+);

database/migrations/20251207020207_favicons.down.sql

@@ -0,0 +1 @@
+DROP TABLE favicons;

database/migrations/20251207020207_favicons.up.sql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS favicons
+(
+    id   INTEGER PRIMARY KEY AUTO_INCREMENT,
+    host TEXT NOT NULL,
+    svg  TEXT,
+    png  TEXT,
+    ico  TEXT
+);

database/migrations/20251207020211_routes.down.sql

@@ -0,0 +1 @@
+DROP TABLE routes;

database/migrations/20251207020211_routes.up.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS routes
+(
+    id          INTEGER PRIMARY KEY AUTO_INCREMENT,
+    source      TEXT UNIQUE NOT NULL,
+    destination TEXT        NOT NULL,
+    description TEXT        NOT NULL,
+    flags       INTEGER     NOT NULL DEFAULT 0,
+    active      BOOLEAN     NOT NULL DEFAULT 1
+);

database/migrations/20251207020214_redirects.down.sql

@@ -0,0 +1 @@
+DROP TABLE redirects;

database/migrations/20251207020214_redirects.up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS redirects
+(
+    id          INTEGER PRIMARY KEY AUTO_INCREMENT,
+    source      TEXT UNIQUE NOT NULL,
+    destination TEXT        NOT NULL,
+    description TEXT        NOT NULL,
+    flags       INTEGER     NOT NULL DEFAULT 0,
+    code        INTEGER     NOT NULL DEFAULT 0,
+    active      BOOLEAN     NOT NULL DEFAULT 1
+);

database/queries/domain.sql

@@ -4,13 +4,11 @@ FROM domains
 WHERE active = 1;
 
 -- name: AddDomain :exec
-INSERT OR
 REPLACE
 INTO domains (domain, active)
 VALUES (?, ?);
 
 -- name: DeleteDomain :exec
-INSERT OR
 REPLACE
 INTO domains(domain, active)
 VALUES (?, false);