Merge pull request #181 from SiaFoundation/christopher/fix-v2-storage-total

Add v2 numbers to total and remaining storage in metrics

Author: chris124567

api/api_test.go

@@ -49,8 +49,9 @@ func newExplorer(t *testing.T, network *consensus.Network, genesisBlock types.Bl
 	cm := chain.NewManager(store, genesisState)
 
 	e, err := explorer.NewExplorer(cm, db, 1000, config.Scanner{
-		Threads:             10,
+		BatchSize:           100,
 		Timeout:             30 * time.Second,
+		CheckAgainDelay:     100 * time.Millisecond,
 		MaxLastScan:         3 * time.Hour,
 		MinLastAnnouncement: 90 * 24 * time.Hour,
 	}, log)

cmd/explored/main.go

@@ -45,8 +45,9 @@ var cfg = config.Config{
 		EnableUPNP: false,
 	},
 	Scanner: config.Scanner{
-		Threads:             10,
+		BatchSize:           100,
 		Timeout:             1 * time.Minute,
+		CheckAgainDelay:     15 * time.Second,
 		MaxLastScan:         1 * time.Hour,
 		MinLastAnnouncement: 365 * 24 * time.Hour,
 	},

config/config.go

@@ -18,9 +18,24 @@ type (
 
 	// Scanner contains the configuration for the host scanner.
 	Scanner struct {
-		Threads             int           `yaml:"threads,omitempty"`
-		Timeout             time.Duration `yaml:"timeout,omitempty"`
-		MaxLastScan         time.Duration `yaml:"maxLastScan,omitempty"`
+		// BatchSize represents the maximum number of hosts we will
+		// simultaneously scan.
+		BatchSize uint64 `yaml:"batchSize,omitempty"`
+		// Timeout represents the maximum amount of time we will spend scanning
+		// a single host.
+		Timeout time.Duration `yaml:"timeout,omitempty"`
+		// CheckAgainDelay represents the amount of time we will wait before
+		// calling HostsForScanning again if the previous call returned zero
+		// hosts to scan.
+		CheckAgainDelay time.Duration `yaml:"checkAgainDelay,omitempty"`
+		// MaxLastScan represents how frequently hosts will be scanned.  If a
+		// scan is successful, the hosts next scan time will be set to
+		// the current time plus MaxLastScan.  If it fails, the next scan time
+		// is set to the current time plus MaxLastScan * pow(2, # of
+		// consecutive failed scans).
+		MaxLastScan time.Duration `yaml:"maxLastScan,omitempty"`
+		// MinLastAnnouncement represents how far back we will search for
+		// announcements to find hosts to scan.
 		MinLastAnnouncement time.Duration `yaml:"minLastAnnouncement,omitempty"`
 	}
 

explorer/scan.go

@@ -18,10 +18,6 @@ import (
 	"go.uber.org/zap"
 )
 
-const (
-	scanBatchSize = 100
-)
-
 func isSynced(b Block) bool {
 	return time.Since(b.Timestamp) <= 3*time.Hour
 }
@@ -226,7 +222,7 @@ func (e *Explorer) scanHosts() {
 		now := types.CurrentTimestamp()
 		lastAnnouncementCutoff := now.Add(-e.scanCfg.MinLastAnnouncement)
 
-		batch, err := e.s.HostsForScanning(lastAnnouncementCutoff, scanBatchSize)
+		batch, err := e.s.HostsForScanning(lastAnnouncementCutoff, e.scanCfg.BatchSize)
 		if err != nil {
 			e.log.Info("failed to get hosts for scanning:", zap.Error(err))
 			return
@@ -235,7 +231,8 @@ func (e *Explorer) scanHosts() {
 			case <-e.ctx.Done():
 				e.log.Info("shutdown:", zap.Error(e.ctx.Err()))
 				return
-			case <-time.After(15 * time.Second):
+			// wait until we call HostsForScanning again
+			case <-time.After(e.scanCfg.CheckAgainDelay):
 				continue // check again
 			}
 		}

persist/sqlite/metrics.go

@@ -5,6 +5,8 @@ import (
 	"slices"
 	"time"
 
+	proto4 "go.sia.tech/core/rhp/v4"
+
 	"go.sia.tech/core/types"
 	"go.sia.tech/explored/explorer"
 )
@@ -65,10 +67,27 @@ func (s *Store) HostMetrics() (result explorer.HostMetrics, err error) {
 				return fmt.Errorf("failed to scan host: %w", err)
 			}
 
-			result.TotalStorage += host.Settings.TotalStorage
-			result.RemainingStorage += host.Settings.RemainingStorage
+			if host.V2 {
+				result.TotalStorage += proto4.SectorSize * host.V2Settings.TotalStorage
+				result.RemainingStorage += proto4.SectorSize * host.V2Settings.RemainingStorage
+
+				v2MaxCollateral = append(v2MaxCollateral, host.V2Settings.MaxCollateral)
+				v2MaxContractDuration = append(v2MaxContractDuration, host.V2Settings.MaxContractDuration)
+				v2RemainingStorage = append(v2RemainingStorage, host.V2Settings.RemainingStorage)
+				v2TotalStorage = append(v2TotalStorage, host.V2Settings.TotalStorage)
+
+				v2PricesContractPrice = append(v2PricesContractPrice, host.V2Settings.Prices.ContractPrice)
+				v2PricesCollateral = append(v2PricesCollateral, host.V2Settings.Prices.Collateral)
+				v2PricesStoragePrice = append(v2PricesStoragePrice, host.V2Settings.Prices.StoragePrice)
+				v2PricesIngressPrice = append(v2PricesIngressPrice, host.V2Settings.Prices.IngressPrice)
+				v2PricesEgressPrice = append(v2PricesEgressPrice, host.V2Settings.Prices.EgressPrice)
+				v2PricesFreeSectorPrice = append(v2PricesFreeSectorPrice, host.V2Settings.Prices.FreeSectorPrice)
+				v2PricesTipHeight = append(v2PricesTipHeight, host.V2Settings.Prices.TipHeight)
+				v2PricesValidUntil = append(v2PricesValidUntil, uint64(host.V2Settings.Prices.ValidUntil.Unix()))
+			} else {
+				result.TotalStorage += host.Settings.TotalStorage
+				result.RemainingStorage += host.Settings.RemainingStorage
 
-			if !host.V2 {
 				settingsMaxDownloadBatchSize = append(settingsMaxDownloadBatchSize, host.Settings.MaxDownloadBatchSize)
 				settingsMaxDuration = append(settingsMaxDuration, host.Settings.MaxDuration)
 				settingsMaxReviseBatchSize = append(settingsMaxReviseBatchSize, host.Settings.MaxReviseBatchSize)
@@ -120,20 +139,6 @@ func (s *Store) HostMetrics() (result explorer.HostMetrics, err error) {
 				priceTableWindowSize = append(priceTableWindowSize, host.PriceTable.WindowSize)
 				priceTableRegistryEntriesLeft = append(priceTableRegistryEntriesLeft, host.PriceTable.RegistryEntriesLeft)
 				priceTableRegistryEntriesTotal = append(priceTableRegistryEntriesTotal, host.PriceTable.RegistryEntriesTotal)
-			} else {
-				v2MaxCollateral = append(v2MaxCollateral, host.V2Settings.MaxCollateral)
-				v2MaxContractDuration = append(v2MaxContractDuration, host.V2Settings.MaxContractDuration)
-				v2RemainingStorage = append(v2RemainingStorage, host.V2Settings.RemainingStorage)
-				v2TotalStorage = append(v2TotalStorage, host.V2Settings.TotalStorage)
-
-				v2PricesContractPrice = append(v2PricesContractPrice, host.V2Settings.Prices.ContractPrice)
-				v2PricesCollateral = append(v2PricesCollateral, host.V2Settings.Prices.Collateral)
-				v2PricesStoragePrice = append(v2PricesStoragePrice, host.V2Settings.Prices.StoragePrice)
-				v2PricesIngressPrice = append(v2PricesIngressPrice, host.V2Settings.Prices.IngressPrice)
-				v2PricesEgressPrice = append(v2PricesEgressPrice, host.V2Settings.Prices.EgressPrice)
-				v2PricesFreeSectorPrice = append(v2PricesFreeSectorPrice, host.V2Settings.Prices.FreeSectorPrice)
-				v2PricesTipHeight = append(v2PricesTipHeight, host.V2Settings.Prices.TipHeight)
-				v2PricesValidUntil = append(v2PricesValidUntil, uint64(host.V2Settings.Prices.ValidUntil.Unix()))
 			}
 
 			count++