From 7f95036c2db9283c05030105d5a90aae770716e0 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Thu, 6 Nov 2025 13:22:05 +0100
Subject: [PATCH 1/8] cmd/sunlight,internal/witness: pull witness config from
 witness network log list

---
 cmd/sunlight/home.html      |  44 ++---
 cmd/sunlight/sunlight.go    |  65 +++++---
 internal/witness/witness.go | 314 +++++++++++++++++++++++++++++-------
 3 files changed, 329 insertions(+), 94 deletions(-)

diff --git a/cmd/sunlight/home.html b/cmd/sunlight/home.html
index 3b9fa4e..32a1544 100644
--- a/cmd/sunlight/home.html
+++ b/cmd/sunlight/home.html
@@ -52,26 +52,6 @@
     
     <p>
       <a href="metrics">Metrics</a> are available.
-    
-    {{ if .Witness.Name }}
-    <hr>
-
-    <p>
-      The following witness is active.
-
-    <h2>{{ .Witness.Name }}</h2>
-
-    <p>
-      Submission prefix: <code>{{ .Witness.SubmissionPrefix }}</code><br>
-      Known logs:
-      <ul>
-        {{ range .Witness.Logs }}
-        <li><code>{{ . }}</code></li>
-        {{ end }}
-      </ul>
-
-    <pre><code>{{ .Witness.VerifierKey }}</code></pre>
-    {{ end }}
 
     <hr>
 
@@ -103,6 +83,30 @@ <h3>Submit a certificate chain (PEM or JSON)</h3>
 
     {{ end }}
 
+    {{ if .Witness.Name }}
+    <hr>
+
+    <p>
+      The following witness is active.
+
+    <h2>{{ .Witness.Name }}</h2>
+
+    <p>
+      Submission prefix: <code>{{ .Witness.SubmissionPrefix }}</code><br>
+      Log list source: <a href="{{ .Witness.LogList }}">{{ .Witness.LogList }}</a><br>
+      Verifier key:<br>
+
+    <pre><code>{{ .Witness.VerifierKey }}</code></pre>
+
+    <p>
+      Known logs:
+      <ul>
+        {{ range .Witness.Logs }}
+        <li><code>{{ . }}</code></li>
+        {{ end }}
+      </ul>
+    {{ end }}
+
     <script>
         for (const fileInput of document.querySelectorAll('.chain')) {
             fileInput.addEventListener('change', async (event) => {
diff --git a/cmd/sunlight/sunlight.go b/cmd/sunlight/sunlight.go
index 90e826f..de507e9 100644
--- a/cmd/sunlight/sunlight.go
+++ b/cmd/sunlight/sunlight.go
@@ -149,9 +149,11 @@ type Config struct {
 		//
 		Secret string
 
-		// KnownLogs is a list of known logs that the witness will accept and
-		// cosign checkpoints for, along with their vkeys.
-		KnownLogs []witness.LogConfig
+		// LogList is the URL of a witness network log list.
+		//
+		// Logs are pulled from the list on startup and every 15 minutes. The
+		// list can only add logs, never remove them or change their public key.
+		LogList string
 	}
 
 	Logs []LogConfig
@@ -413,18 +415,24 @@ func main() {
 
 	sequencerGroup, sequencerContext := errgroup.WithContext(ctx)
 
-	var homeData struct {
-		Logs    []logInfo
-		Witness struct {
-			Name             string
-			SubmissionPrefix string
-			VerifierKey      string
-			Logs             []string
-		}
+	var homeLogsInfo []logInfo
+	type witnessInfo struct {
+		Name             string
+		SubmissionPrefix string
+		VerifierKey      string
+		LogList          string
+		Logs             []string
 	}
+	homeWitnessInfo := func() witnessInfo { return witnessInfo{} }
 	mux.HandleFunc("/{$}", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/html")
-		if err := homeTmpl.Execute(w, homeData); err != nil {
+		if err := homeTmpl.Execute(w, struct {
+			Logs    []logInfo
+			Witness witnessInfo
+		}{
+			Logs:    homeLogsInfo,
+			Witness: homeWitnessInfo(),
+		}); err != nil {
 			logger.Error("failed to execute homepage template", "err", err)
 		}
 	})
@@ -644,7 +652,7 @@ func main() {
 		}
 		log.Interval.NotAfterStart = lc.NotAfterStart
 		log.Interval.NotAfterLimit = lc.NotAfterLimit
-		homeData.Logs = append(homeData.Logs, log)
+		homeLogsInfo = append(homeLogsInfo, log)
 
 		j, err := json.MarshalIndent(log, "", "    ")
 		if err != nil {
@@ -685,12 +693,28 @@ func main() {
 			Key:     wk,
 			Backend: db,
 			Log:     logger,
-			Logs:    c.Witness.KnownLogs,
 		})
 		if err != nil {
 			fatalError(logger, "failed to create witness", "err", err)
 		}
 
+		reloadChan := make(chan os.Signal, 1)
+		signal.Notify(reloadChan, syscall.SIGHUP)
+		serveGroup.Go(func() error {
+			ticker := time.NewTicker(15 * time.Minute)
+			for {
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-reloadChan:
+				case <-ticker.C:
+				}
+				if err := w.PullLogList(ctx, c.Witness.LogList); err != nil {
+					logger.Error("failed to pull log list", "err", err)
+				}
+			}
+		})
+
 		c.Witness.SubmissionPrefix = strings.TrimSuffix(c.Witness.SubmissionPrefix, "/")
 		prefix, err := url.Parse(c.Witness.SubmissionPrefix)
 		if err != nil {
@@ -711,11 +735,14 @@ func main() {
 		witnessMetrics := prometheus.WrapRegistererWithPrefix("witness_", sunlightMetrics)
 		witnessMetrics.MustRegister(w.Metrics()...)
 
-		homeData.Witness.Name = c.Witness.Name
-		homeData.Witness.SubmissionPrefix = c.Witness.SubmissionPrefix
-		homeData.Witness.VerifierKey = w.VerifierKey()
-		for _, log := range c.Witness.KnownLogs {
-			homeData.Witness.Logs = append(homeData.Witness.Logs, log.Origin)
+		homeWitnessInfo = func() witnessInfo {
+			return witnessInfo{
+				Name:             c.Witness.Name,
+				SubmissionPrefix: c.Witness.SubmissionPrefix,
+				VerifierKey:      w.VerifierKey(),
+				LogList:          c.Witness.LogList,
+				Logs:             w.Logs(),
+			}
 		}
 	}
 
diff --git a/internal/witness/witness.go b/internal/witness/witness.go
index c9c3d3f..d78d98b 100644
--- a/internal/witness/witness.go
+++ b/internal/witness/witness.go
@@ -3,14 +3,18 @@ package witness
 import (
 	"bytes"
 	"context"
+	"crypto"
 	"crypto/ed25519"
 	"crypto/sha256"
 	"encoding/asn1"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
+	"maps"
 	"net/http"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -25,53 +29,94 @@ import (
 
 type Config struct {
 	Name string
-	Key  ed25519.PrivateKey
+	Key  crypto.Signer
 
 	Backend ctlog.LockBackend
 	Log     *slog.Logger
-
-	Logs []LogConfig
-}
-
-type LogConfig struct {
-	// Origin is the fully qualified log name for the checkpoint origin
-	// line, usually a schema-less URL.
-	Origin string
-
-	// VerifierKeys is a list of vkey strings for the log.
-	VerifierKeys []string
 }
 
-func logIDFromOrigin(config *Config, origin string) [sha256.Size]byte {
+// backendKeyForCheckpoint computes the LockBackend key for the given log origin.
+//
+// It is domain separated from all uses of LockBackend by the ctlog package, and
+// by other Witness instances.
+func backendKeyForCheckpoint(config *Config, origin string) [sha256.Size]byte {
 	h := sha256.New()
 
 	// Domain separation from [ctlog.logIDFromKey].
 	h.Write(asn1.NullBytes)
-	h.Write([]byte("Sunlight witness\n"))
+	h.Write([]byte("witness log\n"))
 
 	// Let multiple witnesses share the same LockBackend without affecting each
-	// other's state. This is undesirable for logs we operate, where we are in
-	// charge of preventing split-views, but for witnesses it would mostly cause
-	// conflicts as it would invalidate the client's view of the witness state.
-	h.Write([]byte(config.Name))
-	h.Write([]byte("\n"))
+	// other's state. This is the opposite of what we want for logs we operate,
+	// where we are in charge of preventing split-views, but witnesses have each
+	// their own view of the state of each log they witness.
+	//
+	// Use the key instead of the name to prevent multiple witnesses from being
+	// erroneously configured with the same key.
+	h.Write(config.Key.Public().(ed25519.PublicKey))
 
 	h.Write([]byte(origin))
 	return [32]byte(h.Sum(nil))
 }
 
+// backendKeyForConfig computes the LockBackend key for the witness
+// configuration.
+func backendKeyForConfig(config *Config) [sha256.Size]byte {
+	h := sha256.New()
+
+	// Domain separation from all other uses of LockBackend.
+	h.Write(asn1.NullBytes)
+	h.Write([]byte("witness config\n"))
+
+	h.Write(config.Key.Public().(ed25519.PublicKey))
+	return [32]byte(h.Sum(nil))
+}
+
 type Witness struct {
 	c *Config
 	s *torchwood.CosignatureSigner
 	m metrics
 
-	verifiers map[string]note.Verifiers
-	locks     map[string]*lockedCheckpoint
+	// verifiers and checkpoints are indexed by log origin. They must be
+	// accessed and updated under logsMu. The list of origins and their verifier
+	// keys are stored in Backend, and are updated additively by PullLogList.
+	logsMu      sync.RWMutex
+	verifiers   map[string]note.Verifiers
+	checkpoints map[string]*lockedCheckpoint
+}
+
+func (w *Witness) verifiersForOrigin(origin string) (note.Verifiers, bool) {
+	w.logsMu.RLock()
+	defer w.logsMu.RUnlock()
+	v, ok := w.verifiers[origin]
+	return v, ok
+}
+
+func (w *Witness) checkpointForOrigin(origin string) (*lockedCheckpoint, bool) {
+	w.logsMu.RLock()
+	defer w.logsMu.RUnlock()
+	c, ok := w.checkpoints[origin]
+	return c, ok
+}
+
+func (w *Witness) Logs() []string {
+	w.logsMu.RLock()
+	defer w.logsMu.RUnlock()
+	logs := slices.Collect(maps.Keys(w.verifiers))
+	slices.Sort(logs)
+	return logs
+}
+
+type storedConfig struct {
+	Logs []struct {
+		Origin       string   `json:"origin"`
+		VerifierKeys []string `json:"verifierKeys"`
+	} `json:"logs"`
 }
 
 type lockedCheckpoint struct {
 	sync.Mutex
-	ctlog.LockedCheckpoint
+	ctlog.LockedCheckpoint // can be nil if not fetched yet
 }
 
 func NewWitness(ctx context.Context, config *Config) (*Witness, error) {
@@ -80,12 +125,31 @@ func NewWitness(ctx context.Context, config *Config) (*Witness, error) {
 		return nil, fmt.Errorf("couldn't create signer: %w", err)
 	}
 
-	l := make(map[string]note.Verifiers, len(config.Logs))
-	locks := make(map[string]*lockedCheckpoint)
-	for _, log := range config.Logs {
-		if _, ok := l[log.Origin]; ok {
+	w := &Witness{c: config, s: s, m: initMetrics()}
+
+	logs, err := config.Backend.Fetch(ctx, backendKeyForConfig(config))
+	if errors.Is(err, ctlog.ErrLogNotFound) {
+		config.Log.WarnContext(ctx, "witness config not found in backend; creating new empty config")
+		if err := config.Backend.Create(ctx, backendKeyForConfig(config), []byte("{}")); err != nil {
+			return nil, fmt.Errorf("couldn't create witness config in backend: %w", err)
+		}
+		logs, err = config.Backend.Fetch(ctx, backendKeyForConfig(config))
+	}
+	if err != nil {
+		return nil, fmt.Errorf("couldn't fetch witness config from backend: %w", err)
+	}
+	var stored storedConfig
+	if err := json.Unmarshal(logs.Bytes(), &stored); err != nil {
+		return nil, fmt.Errorf("couldn't parse stored witness config: %w", err)
+	}
+
+	w.verifiers = make(map[string]note.Verifiers, len(stored.Logs))
+	w.checkpoints = make(map[string]*lockedCheckpoint, len(stored.Logs))
+	for _, log := range stored.Logs {
+		if _, ok := w.verifiers[log.Origin]; ok {
 			return nil, fmt.Errorf("two logs with origin %q", log.Origin)
 		}
+		config.Log.DebugContext(ctx, "configured to witness log", "origin", log.Origin, "vkeys", log.VerifierKeys)
 		var verifiers []note.Verifier
 		for _, k := range log.VerifierKeys {
 			v, err := note.NewVerifier(k)
@@ -94,17 +158,12 @@ func NewWitness(ctx context.Context, config *Config) (*Witness, error) {
 			}
 			verifiers = append(verifiers, v)
 		}
-		l[log.Origin] = note.VerifierList(verifiers...)
-		c, err := config.Backend.Fetch(ctx, logIDFromOrigin(config, log.Origin))
-		if err != nil && !errors.Is(err, ctlog.ErrLogNotFound) {
-			return nil, fmt.Errorf("couldn't fetch checkpoint for log %q: %w", log.Origin, err)
-		}
-		locks[log.Origin] = &lockedCheckpoint{LockedCheckpoint: c}
+		w.verifiers[log.Origin] = note.VerifierList(verifiers...)
+		// Defer fetching checkpoints until needed, to avoid blocking startup.
+		w.checkpoints[log.Origin] = &lockedCheckpoint{}
 	}
 
-	config.Log.InfoContext(ctx, "starting witness", "name", config.Name, "numLogs", len(config.Logs))
-
-	w := &Witness{c: config, s: s, verifiers: l, locks: locks, m: initMetrics()}
+	config.Log.InfoContext(ctx, "starting witness", "name", config.Name, "logs", len(stored.Logs))
 	return w, nil
 }
 
@@ -112,6 +171,88 @@ func (w *Witness) VerifierKey() string {
 	return w.s.Verifier().String()
 }
 
+func (w *Witness) PullLogList(ctx context.Context, url string) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create log list request: %w", err)
+	}
+	req.Header.Set("User-Agent", "+https://filippo.io/sunlight")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to fetch log list: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to fetch log list: %s", resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read log list response: %w", err)
+	}
+	logs, err := parseLogList(body)
+	if err != nil {
+		return fmt.Errorf("failed to parse log list: %w", err)
+	}
+
+	w.logsMu.Lock()
+	defer w.logsMu.Unlock()
+	verifiers := maps.Clone(w.verifiers)
+	checkpoints := maps.Clone(w.checkpoints)
+	newLogsAndVkeys := make(map[string]string)
+	for origin, vkey := range logs {
+		if _, ok := verifiers[origin]; ok {
+			// Already known, skip.
+			continue
+		}
+		v, err := note.NewVerifier(vkey)
+		if err != nil {
+			return fmt.Errorf("couldn't parse vkey %q for log %q: %w", vkey, origin, err)
+		}
+		w.c.Log.InfoContext(ctx, "adding new log to witness config", "origin", origin, "vkey", vkey)
+		if err := w.c.Backend.Create(ctx, backendKeyForCheckpoint(w.c, origin), nil); err != nil {
+			return fmt.Errorf("couldn't create empty checkpoint for new log %q: %w", origin, err)
+		}
+		ch, err := w.c.Backend.Fetch(ctx, backendKeyForCheckpoint(w.c, origin))
+		if err != nil {
+			return fmt.Errorf("couldn't fetch empty checkpoint for new log %q: %w", origin, err)
+		}
+		verifiers[origin] = note.VerifierList(v)
+		checkpoints[origin] = &lockedCheckpoint{LockedCheckpoint: ch}
+		newLogsAndVkeys[origin] = vkey
+	}
+
+	// We don't actually need the compare-and-swap semantics to update the list
+	// of verifier keys, a rollback of that would not cause a checkpoint rollback.
+	oldLogs, err := w.c.Backend.Fetch(ctx, backendKeyForConfig(w.c))
+	if err != nil {
+		return fmt.Errorf("couldn't fetch existing witness config from backend: %w", err)
+	}
+	var stored storedConfig
+	if err := json.Unmarshal(oldLogs.Bytes(), &stored); err != nil {
+		return fmt.Errorf("couldn't parse stored witness config: %w", err)
+	}
+	for origin, vkey := range newLogsAndVkeys {
+		stored.Logs = append(stored.Logs, struct {
+			Origin       string   `json:"origin"`
+			VerifierKeys []string `json:"verifierKeys"`
+		}{
+			Origin:       origin,
+			VerifierKeys: []string{vkey},
+		})
+	}
+	newConfigBytes, err := json.Marshal(stored)
+	if err != nil {
+		return fmt.Errorf("couldn't marshal updated witness config: %w", err)
+	}
+	if _, err := w.c.Backend.Replace(ctx, oldLogs, newConfigBytes); err != nil {
+		return fmt.Errorf("couldn't store updated witness config: %w", err)
+	}
+	w.verifiers = verifiers
+	w.checkpoints = checkpoints
+
+	return nil
+}
+
 func (w *Witness) Handler() http.Handler {
 	labels := prometheus.Labels{"endpoint": "get-roots"}
 	addCheckpoint := http.Handler(http.HandlerFunc(w.serveAddCheckpoint))
@@ -210,7 +351,7 @@ func (w *Witness) processAddCheckpointRequest(ctx context.Context, body []byte)
 	}
 	origin, _, _ := strings.Cut(string(noteBytes), "\n")
 	l = l.With("origin", origin)
-	v, ok := w.verifiers[origin]
+	v, ok := w.verifiersForOrigin(origin)
 	if !ok {
 		return nil, errUnknownLog
 	}
@@ -237,7 +378,7 @@ func (w *Witness) updateCheckpoint(ctx context.Context, origin string,
 	oldSize, newSize int64, newHash tlog.Hash, proof tlog.TreeProof,
 	noteBytes []byte) ([]byte, error) {
 
-	lock, ok := w.locks[origin]
+	lock, ok := w.checkpointForOrigin(origin)
 	if !ok {
 		return nil, errors.New("internal error: lock not found for known log")
 	}
@@ -245,6 +386,16 @@ func (w *Witness) updateCheckpoint(ctx context.Context, origin string,
 	defer lock.Unlock()
 
 	if lock.LockedCheckpoint == nil {
+		// Might not have been fetched yet, do it now. It must exist because it
+		// is created (empty) when the log is added to the witness config.
+		c, err := w.c.Backend.Fetch(ctx, backendKeyForCheckpoint(w.c, origin))
+		if err != nil {
+			return nil, errors.New("internal error: couldn't fetch checkpoint for known log")
+		}
+		lock.LockedCheckpoint = c
+	}
+
+	if len(lock.Bytes()) == 0 {
 		if oldSize != 0 {
 			return nil, &conflictError{0}
 		}
@@ -288,25 +439,12 @@ func (w *Witness) updateCheckpoint(ctx context.Context, origin string,
 	}
 	new := append(noteBytes[:len(noteBytes):len(noteBytes)], sigs...)
 
-	if lock.LockedCheckpoint == nil {
-		err := w.c.Backend.Create(ctx, logIDFromOrigin(w.c, origin), new)
-		if err != nil {
-			return nil, errors.New("internal error: failed to create new checkpoint")
-		}
-		// Kinda unclear why [ctlog.LockBackend.Create] doesn't return the
-		// [ctlog.LockedCheckpoint], but a race here would be harmless anyway.
-		newLock, err := w.c.Backend.Fetch(ctx, logIDFromOrigin(w.c, origin))
-		if err != nil {
-			return nil, errors.New("internal error: failed to fetch new checkpoint")
-		}
-		lock.LockedCheckpoint = newLock
-	} else {
-		newLock, err := w.c.Backend.Replace(ctx, lock.LockedCheckpoint, new)
-		if err != nil {
-			return nil, errors.New("internal error: failed to store new checkpoint")
-		}
-		lock.LockedCheckpoint = newLock
+	newLock, err := w.c.Backend.Replace(ctx, lock.LockedCheckpoint, new)
+	if err != nil {
+		// TODO: this is a fatal error, because we don't know if it was persisted.
+		return nil, errors.New("internal error: failed to store new checkpoint")
 	}
+	lock.LockedCheckpoint = newLock
 
 	return sigs, nil
 }
@@ -323,3 +461,69 @@ func splitSignatures(note []byte) ([]byte, error) {
 	}
 	return sigs, nil
 }
+
+func parseLogList(logList []byte) (map[string]string, error) {
+	logs := make(map[string]string)
+	var sawHeader bool
+	var vkey, origin string
+	finalizeLogEntry := func() {
+		if vkey == "" {
+			// The list may be empty.
+			return
+		}
+		defer func() {
+			vkey = ""
+			origin = ""
+		}()
+		v, err := note.NewVerifier(vkey)
+		if err != nil {
+			// Invalid vkey, skip.
+			return
+		}
+		if origin == "" {
+			origin = v.Name()
+		}
+		if logs[origin] != "" {
+			// Duplicate origin, skip.
+			return
+		}
+		logs[origin] = vkey
+	}
+	for line := range strings.Lines(string(logList)) {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "#") {
+			// Comment line, skip.
+			continue
+		}
+		if line == "" {
+			// Empty line, skip.
+			continue
+		}
+		if !sawHeader {
+			// First non-comment, non-empty line is the header.
+			if line != "logs/v0" {
+				return nil, fmt.Errorf("invalid log list header: %q", line)
+			}
+			sawHeader = true
+			continue
+		}
+		key, value, _ := strings.Cut(line, " ")
+		if vkey == "" && key != "vkey" {
+			return nil, fmt.Errorf("expected vkey entry, got %q", line)
+		}
+		switch key {
+		case "vkey":
+			finalizeLogEntry()
+			if value == "" {
+				return nil, fmt.Errorf("empty vkey entry")
+			}
+			vkey = value
+		case "origin":
+			origin = value
+		default:
+			// Unknown key, ignore.
+		}
+	}
+	finalizeLogEntry()
+	return logs, nil
+}

From 685bf6e779f9b205986f7b8a90b5f7be8b0f83e8 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Thu, 6 Nov 2025 17:55:11 +0100
Subject: [PATCH 2/8] internal/witness: add per-log witness metrics

---
 internal/witness/metrics.go | 26 +++++++++++++++++++++++---
 internal/witness/witness.go | 25 ++++++++++++++++---------
 2 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/internal/witness/metrics.go b/internal/witness/metrics.go
index d6cba46..5f5cdd6 100644
--- a/internal/witness/metrics.go
+++ b/internal/witness/metrics.go
@@ -7,10 +7,11 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-// TODO: add-checkpoint metrics partitioned by origin, outcome.
-// TODO: log size gauge partitioned by origin.
-
 type metrics struct {
+	KnownLogs          prometheus.Gauge
+	LogSize            *prometheus.GaugeVec
+	AddCheckpointCount *prometheus.CounterVec
+
 	ReqCount    *prometheus.CounterVec
 	ReqInFlight *prometheus.GaugeVec
 	ReqDuration *prometheus.SummaryVec
@@ -18,6 +19,25 @@ type metrics struct {
 
 func initMetrics() metrics {
 	return metrics{
+		KnownLogs: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "known_logs",
+			Help: "Number of logs known to the witness.",
+		}),
+		LogSize: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "log_entries_total",
+				Help: "Size of the latest checkpoint, by log origin.",
+			},
+			[]string{"origin"},
+		),
+		AddCheckpointCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "add_checkpoint_requests_total",
+				Help: "Total number of add-checkpoint requests processed, by log origin.",
+			},
+			[]string{"error", "origin", "progress"},
+		),
+
 		ReqInFlight: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "http_in_flight_requests",
diff --git a/internal/witness/witness.go b/internal/witness/witness.go
index d78d98b..70189bb 100644
--- a/internal/witness/witness.go
+++ b/internal/witness/witness.go
@@ -163,6 +163,7 @@ func NewWitness(ctx context.Context, config *Config) (*Witness, error) {
 		w.checkpoints[log.Origin] = &lockedCheckpoint{}
 	}
 
+	w.m.KnownLogs.Set(float64(len(stored.Logs)))
 	config.Log.InfoContext(ctx, "starting witness", "name", config.Name, "logs", len(stored.Logs))
 	return w, nil
 }
@@ -249,12 +250,13 @@ func (w *Witness) PullLogList(ctx context.Context, url string) error {
 	}
 	w.verifiers = verifiers
 	w.checkpoints = checkpoints
+	w.m.KnownLogs.Set(float64(len(w.verifiers)))
 
 	return nil
 }
 
 func (w *Witness) Handler() http.Handler {
-	labels := prometheus.Labels{"endpoint": "get-roots"}
+	labels := prometheus.Labels{"endpoint": "add-checkpoint"}
 	addCheckpoint := http.Handler(http.HandlerFunc(w.serveAddCheckpoint))
 	addCheckpoint = promhttp.InstrumentHandlerCounter(w.m.ReqCount.MustCurryWith(labels), addCheckpoint)
 	addCheckpoint = promhttp.InstrumentHandlerDuration(w.m.ReqDuration.MustCurryWith(labels), addCheckpoint)
@@ -274,6 +276,7 @@ func (*conflictError) Error() string { return "known tree size doesn't match pro
 var errUnknownLog = errors.New("unknown log")
 var errInvalidSignature = errors.New("invalid signature")
 var errBadRequest = errors.New("invalid input")
+var errBadCheckpoint = errors.New("invalid checkpoint")
 var errProof = errors.New("bad consistency proof")
 
 func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) {
@@ -318,12 +321,12 @@ func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) {
 }
 
 func (w *Witness) processAddCheckpointRequest(ctx context.Context, body []byte) (cosig []byte, err error) {
-	l := w.c.Log.With("request", string(body))
+	labels := prometheus.Labels{"error": "", "origin": "", "progress": ""}
 	defer func() {
 		if err != nil {
-			l = l.With("error", err)
+			labels["error"] = err.Error()
 		}
-		l.Debug("processed add-checkpoint request")
+		w.m.AddCheckpointCount.With(labels).Inc()
 	}()
 	body, noteBytes, ok := bytes.Cut(body, []byte("\n\n"))
 	if !ok {
@@ -341,7 +344,6 @@ func (w *Witness) processAddCheckpointRequest(ctx context.Context, body []byte)
 	if err != nil || oldSize < 0 {
 		return nil, errBadRequest
 	}
-	l = l.With("oldSize", oldSize)
 	proof := make(tlog.TreeProof, len(lines[1:]))
 	for i, h := range lines[1:] {
 		proof[i], err = tlog.ParseHash(h)
@@ -350,27 +352,30 @@ func (w *Witness) processAddCheckpointRequest(ctx context.Context, body []byte)
 		}
 	}
 	origin, _, _ := strings.Cut(string(noteBytes), "\n")
-	l = l.With("origin", origin)
 	v, ok := w.verifiersForOrigin(origin)
 	if !ok {
 		return nil, errUnknownLog
 	}
+	labels["origin"] = origin
 	n, err := note.Open(noteBytes, v)
 	switch err.(type) {
 	case *note.UnverifiedNoteError, *note.InvalidSignatureError:
 		return nil, errInvalidSignature
 	}
 	if err != nil {
-		return nil, err
+		return nil, errors.New("internal error: failed to verify note")
 	}
 	c, err := torchwood.ParseCheckpoint(n.Text)
 	if err != nil {
-		return nil, err
+		return nil, errBadCheckpoint
 	}
 	if origin != c.Origin {
 		return nil, errors.New("internal error: incoherent parsing")
 	}
-	l = l.With("size", c.N)
+	labels["progress"] = "false"
+	if c.N > oldSize {
+		labels["progress"] = "true"
+	}
 	return w.updateCheckpoint(ctx, c.Origin, oldSize, c.N, c.Hash, proof, noteBytes)
 }
 
@@ -446,6 +451,8 @@ func (w *Witness) updateCheckpoint(ctx context.Context, origin string,
 	}
 	lock.LockedCheckpoint = newLock
 
+	w.m.LogSize.WithLabelValues(origin).Set(float64(newSize))
+
 	return sigs, nil
 }
 

From aba19c78befbe3cae5ce82ce1a8289c04a45d435 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Sat, 8 Nov 2025 00:58:22 +0100
Subject: [PATCH 3/8] internal/ctlog: fix SQLite LockBackend for nil values

When passing nil to Create or Replace, it was stored as NULL, but then
was fetched as []byte("") by Fetch, which then didn't compare equal to
NULL in Replace, making it always fail.

This was made harder to debug by two things:

 1. in the CLI, NULL and empty are both shown as empty by default;

 2. I tried to fix it by running

       "UPDATE checkpoints SET body = '' WHERE body IS NULL;"

    which put empty TEXT in those columns, which does not compare equal
    to empty BLOB. They were both allowed in the same column by the
    non-STRICT table.
---
 README.md                |  2 +-
 cmd/sunlight/sunlight.go |  2 +-
 internal/ctlog/cache.go  |  6 +++---
 internal/ctlog/sqlite.go | 11 ++++++++++-
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index dcac23d..fc4f329 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ This database must be global and it **must never be changed or modified** as an
 The database must already exist, to prevent misconfigurations. Create it with 
 
 ```
-sqlite3 checkpoints.db "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body TEXT)"
+sqlite3 checkpoints.db "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body BLOB NOT NULL) STRICT"
 ```
 
 Sunlight can alternatively use DynamoDB or S3-compatible object storage with `ETag` and `If-Match` support (such as Tigris) as global lock backends.
diff --git a/cmd/sunlight/sunlight.go b/cmd/sunlight/sunlight.go
index de507e9..ed647b5 100644
--- a/cmd/sunlight/sunlight.go
+++ b/cmd/sunlight/sunlight.go
@@ -96,7 +96,7 @@ type Config struct {
 	// The database must already exist to protect against accidental
 	// misconfiguration. Create the table with:
 	//
-	//     $ sqlite3 checkpoints.db "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body TEXT)"
+	//     $ sqlite3 checkpoints.db "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body BLOB NOT NULL) STRICT"
 	//
 	Checkpoints string
 
diff --git a/internal/ctlog/cache.go b/internal/ctlog/cache.go
index eff73eb..8e159f8 100644
--- a/internal/ctlog/cache.go
+++ b/internal/ctlog/cache.go
@@ -20,9 +20,9 @@ func initCache(path string) (readConn, writeConn *sqlite.Conn, err error) {
 	if err := sqlitex.ExecTransient(writeConn, `
 		CREATE TABLE IF NOT EXISTS cache (
 			key BLOB PRIMARY KEY,
-			timestamp INTEGER,
-			leaf_index INTEGER
-		) WITHOUT ROWID;`, nil); err != nil {
+			timestamp INTEGER NOT NULL,
+			leaf_index INTEGER NOT NULL
+		) WITHOUT ROWID, STRICT;`, nil); err != nil {
 		writeConn.Close()
 		return nil, nil, err
 	}
diff --git a/internal/ctlog/sqlite.go b/internal/ctlog/sqlite.go
index 653b2d7..6545692 100644
--- a/internal/ctlog/sqlite.go
+++ b/internal/ctlog/sqlite.go
@@ -34,7 +34,7 @@ func NewSQLiteBackend(ctx context.Context, path string, l *slog.Logger) (*SQLite
 
 	conn, err := sqlite.OpenConn(path, sqlite.OpenFlagsDefault & ^sqlite.SQLITE_OPEN_CREATE)
 	if err != nil {
-		return nil, fmt.Errorf(`failed to open SQLite lock database (hint: to avoid misconfiguration, the lock database must be created manually with "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body TEXT)"): %w`, err)
+		return nil, fmt.Errorf(`failed to open SQLite lock database (hint: to avoid misconfiguration, the lock database must be created manually with "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body BLOB NOT NULL) STRICT"): %w`, err)
 	}
 	if err := sqlitex.ExecTransient(conn, "PRAGMA synchronous = FULL", nil); err != nil {
 		conn.Close()
@@ -86,6 +86,10 @@ func (b *SQLiteBackend) Replace(ctx context.Context, old LockedCheckpoint, new [
 	defer prometheus.NewTimer(b.duration).ObserveDuration()
 	b.mu.Lock()
 	defer b.mu.Unlock()
+	if new == nil {
+		// NULL does *not* compare equal to an empty blob!
+		new = []byte{}
+	}
 	o := old.(*sqliteCheckpoint)
 	err := sqlitex.Exec(b.conn, "UPDATE checkpoints SET body = ? WHERE logID = ? AND body = ?",
 		nil, new, o.logID[:], o.body)
@@ -93,6 +97,7 @@ func (b *SQLiteBackend) Replace(ctx context.Context, old LockedCheckpoint, new [
 		return nil, fmtErrorf("failed to update SQLite checkpoint: %w", err)
 	}
 	if b.conn.Changes() == 0 {
+		// TODO: this also hits if new == old.body, and shouldn't.
 		return nil, fmtErrorf("SQLite checkpoint not found or has changed")
 	}
 	return &sqliteCheckpoint{logID: o.logID, body: new}, nil
@@ -101,6 +106,10 @@ func (b *SQLiteBackend) Replace(ctx context.Context, old LockedCheckpoint, new [
 func (b *SQLiteBackend) Create(ctx context.Context, logID [sha256.Size]byte, new []byte) error {
 	b.mu.Lock()
 	defer b.mu.Unlock()
+	if new == nil {
+		// NULL does *not* compare equal to an empty blob!
+		new = []byte{}
+	}
 	err := sqlitex.Exec(b.conn, `INSERT INTO checkpoints (logID, body) VALUES (?, ?)
 		ON CONFLICT(logID) DO NOTHING`, nil, logID[:], new)
 	if err != nil {

From ac1a3bffb5e9d1150f88b39959207f3cb95b7723 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Sat, 8 Nov 2025 01:06:20 +0100
Subject: [PATCH 4/8] internal/witness: prevent swapping witnessed checkpoints
 by backend

---
 internal/witness/witness.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/internal/witness/witness.go b/internal/witness/witness.go
index 70189bb..3149523 100644
--- a/internal/witness/witness.go
+++ b/internal/witness/witness.go
@@ -414,6 +414,9 @@ func (w *Witness) updateCheckpoint(ctx context.Context, origin string,
 			return nil, errors.New("internal error: can't parse stored checkpoint")
 		}
 
+		if known.Origin != origin {
+			return nil, errors.New("internal error: incoherent stored checkpoint")
+		}
 		if oldSize > newSize {
 			return nil, errBadRequest
 		}

From 8f00ab427056216998f95bf36c7e01631ecda9ba Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Sat, 8 Nov 2025 14:04:14 +0100
Subject: [PATCH 5/8] cmd/sunlight: pull roots and logs at start in a goroutine

---
 cmd/sunlight/sunlight.go | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/cmd/sunlight/sunlight.go b/cmd/sunlight/sunlight.go
index ed647b5..f946642 100644
--- a/cmd/sunlight/sunlight.go
+++ b/cmd/sunlight/sunlight.go
@@ -594,10 +594,12 @@ func main() {
 				fatalError(logger, "CCADBRoots must be 'trusted', 'testing', or empty",
 					"CCADBRoots", lc.CCADBRoots)
 			}
-			// We don't run loadCCADBRoots at start, because CCADB is very
-			// flakey, so we don't want to prevent the log from starting if it's
-			// down. The previous roots will be loaded by LoadLog anyway.
 			serveGroup.Go(func() error {
+				if newRoots, err := loadCCADBRoots(ctx, lc, l); err != nil {
+					logger.Error("failed to load initial CCADB roots", "err", err)
+				} else if newRoots {
+					logger.Info("successfully loaded new roots from CCADB/ExtraRoots")
+				}
 				ticker := time.NewTicker(15 * time.Minute)
 				for {
 					select {
@@ -606,12 +608,9 @@ func main() {
 					case <-reloadChan:
 					case <-ticker.C:
 					}
-					newRoots, err := loadCCADBRoots(ctx, lc, l)
-					if err != nil {
+					if newRoots, err := loadCCADBRoots(ctx, lc, l); err != nil {
 						logger.Error("failed to reload CCADB roots", "err", err)
-						continue
-					}
-					if newRoots {
+					} else if newRoots {
 						logger.Info("successfully loaded new roots from CCADB/ExtraRoots on SIGHUP or timer")
 					}
 				}
@@ -701,6 +700,9 @@ func main() {
 		reloadChan := make(chan os.Signal, 1)
 		signal.Notify(reloadChan, syscall.SIGHUP)
 		serveGroup.Go(func() error {
+			if err := w.PullLogList(ctx, c.Witness.LogList); err != nil {
+				logger.Error("failed to pull log list", "err", err)
+			}
 			ticker := time.NewTicker(15 * time.Minute)
 			for {
 				select {

From 26f889fdb704759304627877b21bfeb3c2d7c371 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Thu, 13 Nov 2025 01:31:04 +0100
Subject: [PATCH 6/8] cmd/sunlight: support multiple witness network log lists

---
 cmd/sunlight/home.html      | 15 ++++++++++++---
 cmd/sunlight/sunlight.go    | 24 ++++++++++++++----------
 internal/witness/witness.go |  2 +-
 3 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/cmd/sunlight/home.html b/cmd/sunlight/home.html
index 32a1544..43e7f38 100644
--- a/cmd/sunlight/home.html
+++ b/cmd/sunlight/home.html
@@ -92,12 +92,21 @@ <h3>Submit a certificate chain (PEM or JSON)</h3>
     <h2>{{ .Witness.Name }}</h2>
 
     <p>
-      Submission prefix: <code>{{ .Witness.SubmissionPrefix }}</code><br>
-      Log list source: <a href="{{ .Witness.LogList }}">{{ .Witness.LogList }}</a><br>
-      Verifier key:<br>
+      Submission prefix: <code>{{ .Witness.SubmissionPrefix }}</code>
+
+    <p>
+      Verifier key:
 
     <pre><code>{{ .Witness.VerifierKey }}</code></pre>
 
+    <p>
+      Log list sources:
+      <ul>
+        {{ range .Witness.LogLists }}
+        <li><a href="{{ . }}">{{ . }}</a></li>
+        {{ end }}
+      </ul>
+
     <p>
       Known logs:
       <ul>
diff --git a/cmd/sunlight/sunlight.go b/cmd/sunlight/sunlight.go
index f946642..c8aa040 100644
--- a/cmd/sunlight/sunlight.go
+++ b/cmd/sunlight/sunlight.go
@@ -149,11 +149,11 @@ type Config struct {
 		//
 		Secret string
 
-		// LogList is the URL of a witness network log list.
+		// LogList are the URLs of witness network log list.
 		//
-		// Logs are pulled from the list on startup and every 15 minutes. The
-		// list can only add logs, never remove them or change their public key.
-		LogList string
+		// Logs are pulled from the lists on startup and every 15 minutes. The
+		// lists can only add logs, never remove them or change their public key.
+		LogLists []string
 	}
 
 	Logs []LogConfig
@@ -420,7 +420,7 @@ func main() {
 		Name             string
 		SubmissionPrefix string
 		VerifierKey      string
-		LogList          string
+		LogLists         []string
 		Logs             []string
 	}
 	homeWitnessInfo := func() witnessInfo { return witnessInfo{} }
@@ -700,8 +700,10 @@ func main() {
 		reloadChan := make(chan os.Signal, 1)
 		signal.Notify(reloadChan, syscall.SIGHUP)
 		serveGroup.Go(func() error {
-			if err := w.PullLogList(ctx, c.Witness.LogList); err != nil {
-				logger.Error("failed to pull log list", "err", err)
+			for _, url := range c.Witness.LogLists {
+				if err := w.PullLogList(ctx, url); err != nil {
+					logger.Error("failed to pull log list", "list", url, "err", err)
+				}
 			}
 			ticker := time.NewTicker(15 * time.Minute)
 			for {
@@ -711,8 +713,10 @@ func main() {
 				case <-reloadChan:
 				case <-ticker.C:
 				}
-				if err := w.PullLogList(ctx, c.Witness.LogList); err != nil {
-					logger.Error("failed to pull log list", "err", err)
+				for _, url := range c.Witness.LogLists {
+					if err := w.PullLogList(ctx, url); err != nil {
+						logger.Error("failed to pull log list", "list", url, "err", err)
+					}
 				}
 			}
 		})
@@ -742,7 +746,7 @@ func main() {
 				Name:             c.Witness.Name,
 				SubmissionPrefix: c.Witness.SubmissionPrefix,
 				VerifierKey:      w.VerifierKey(),
-				LogList:          c.Witness.LogList,
+				LogLists:         c.Witness.LogLists,
 				Logs:             w.Logs(),
 			}
 		}
diff --git a/internal/witness/witness.go b/internal/witness/witness.go
index 3149523..8419eaf 100644
--- a/internal/witness/witness.go
+++ b/internal/witness/witness.go
@@ -209,7 +209,7 @@ func (w *Witness) PullLogList(ctx context.Context, url string) error {
 		if err != nil {
 			return fmt.Errorf("couldn't parse vkey %q for log %q: %w", vkey, origin, err)
 		}
-		w.c.Log.InfoContext(ctx, "adding new log to witness config", "origin", origin, "vkey", vkey)
+		w.c.Log.InfoContext(ctx, "adding new log to witness config", "origin", origin, "vkey", vkey, "source", url)
 		if err := w.c.Backend.Create(ctx, backendKeyForCheckpoint(w.c, origin), nil); err != nil {
 			return fmt.Errorf("couldn't create empty checkpoint for new log %q: %w", origin, err)
 		}

From a1e8dd0f0ce33f3811300f6f20019768802037ce Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Sun, 16 Nov 2025 12:03:22 +0100
Subject: [PATCH 7/8] cmd/sunlight: default Witness.SubmissionPrefix to
 "https://" + Name

---
 cmd/sunlight/sunlight.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmd/sunlight/sunlight.go b/cmd/sunlight/sunlight.go
index c8aa040..6ea6423 100644
--- a/cmd/sunlight/sunlight.go
+++ b/cmd/sunlight/sunlight.go
@@ -133,7 +133,8 @@ type Config struct {
 		Name string
 
 		// SubmissionPrefix is the full URL of the c2sp.org/tlog-witness
-		// submission prefix of the witness.
+		// submission prefix of the witness. If not set, it defaults to
+		// "https://" + Name.
 		//
 		// The HTTP server will serve the witness at this URL, and if ACME is
 		// enabled, Sunlight will obtain a certificate for the host of this URL.
@@ -721,6 +722,9 @@ func main() {
 			}
 		})
 
+		if c.Witness.SubmissionPrefix == "" {
+			c.Witness.SubmissionPrefix = "https://" + c.Witness.Name
+		}
 		c.Witness.SubmissionPrefix = strings.TrimSuffix(c.Witness.SubmissionPrefix, "/")
 		prefix, err := url.Parse(c.Witness.SubmissionPrefix)
 		if err != nil {

From 3589bb06cdbaddcb28dfc35ce2ac79aab8edced5 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <hi@filippo.io>
Date: Thu, 27 Nov 2025 16:58:18 +0100
Subject: [PATCH 8/8] cmd/sunlight-keygen: support printing witness verifier
 key

---
 cmd/sunlight-keygen/keygen.go | 17 +++++++++++++++++
 cmd/sunlight/sunlight.go      |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/cmd/sunlight-keygen/keygen.go b/cmd/sunlight-keygen/keygen.go
index 7d4b333..7bc1fd2 100644
--- a/cmd/sunlight-keygen/keygen.go
+++ b/cmd/sunlight-keygen/keygen.go
@@ -17,6 +17,7 @@ import (
 
 	"filippo.io/keygen"
 	"filippo.io/sunlight/internal/immutable"
+	"filippo.io/torchwood"
 	"golang.org/x/crypto/hkdf"
 	"golang.org/x/mod/sumdb/note"
 )
@@ -25,6 +26,7 @@ func main() {
 	fs := flag.NewFlagSet("keygen", flag.ExitOnError)
 	fileFlag := fs.String("f", "", "path to the seed file")
 	prefixFlag := fs.String("prefix", "", "submission prefix for the log, to output a witness verifier key")
+	witnessFlag := fs.String("witness", "", "witness name, for generating a witness secret instead")
 	fs.Parse(os.Args[1:])
 	if fs.NArg() != 0 || *fileFlag == "" {
 		fmt.Fprintln(os.Stderr, "usage: sunlight-keygen -f <seed file>")
@@ -57,6 +59,21 @@ func main() {
 		log.Fatal("seed file must be exactly 32 bytes")
 	}
 
+	if *witnessFlag != "" {
+		ed25519Secret := make([]byte, ed25519.SeedSize)
+		if _, err := io.ReadFull(hkdf.New(sha256.New, seed, []byte("sunlight Ed25519 witness key"),
+			[]byte(*witnessFlag)), ed25519Secret); err != nil {
+			log.Fatal("failed to derive Ed25519 key:", err)
+		}
+		wk := ed25519.NewKeyFromSeed(ed25519Secret)
+		s, err := torchwood.NewCosignatureSigner(*witnessFlag, wk)
+		if err != nil {
+			log.Fatal("failed to create witness signer:", err)
+		}
+		fmt.Printf("Witness vkey: %s\n", s.Verifier())
+		return
+	}
+
 	ecdsaSecret := make([]byte, 32)
 	if _, err := io.ReadFull(hkdf.New(sha256.New, seed, []byte("sunlight"), []byte("ECDSA P-256 log key")), ecdsaSecret); err != nil {
 		log.Fatal("failed to derive ECDSA secret:", err)
diff --git a/cmd/sunlight/sunlight.go b/cmd/sunlight/sunlight.go
index 6ea6423..795dbad 100644
--- a/cmd/sunlight/sunlight.go
+++ b/cmd/sunlight/sunlight.go
@@ -146,7 +146,7 @@ type Config struct {
 		//
 		// To generate a new seed, run:
 		//
-		//   $ sunlight-keygen -f seed.bin
+		//   $ sunlight-keygen -f seed.bin -witness <name>
 		//
 		Secret string