WIP: adding verification for proofs

api/api.go

@@ -22,6 +22,7 @@ import (
 	"github.com/data-preservation-programs/singularity/handler/file"
 	"github.com/data-preservation-programs/singularity/handler/handlererror"
 	"github.com/data-preservation-programs/singularity/handler/job"
+	"github.com/data-preservation-programs/singularity/handler/proof"
 	"github.com/data-preservation-programs/singularity/handler/storage"
 	"github.com/data-preservation-programs/singularity/handler/wallet"
 	"github.com/data-preservation-programs/singularity/model"
@@ -58,6 +59,7 @@ type Server struct {
 	fileHandler     file.Handler
 	jobHandler      job.Handler
 	scheduleHandler schedule.Handler
+	proofHandler    proof.Handler
 }
 
 func Run(c *cli.Context) error {
@@ -138,6 +140,7 @@ func InitServer(ctx context.Context, params APIParams) (*Server, error) {
 		fileHandler:     &file.DefaultHandler{},
 		jobHandler:      &job.DefaultHandler{},
 		scheduleHandler: &schedule.DefaultHandler{},
+		proofHandler:    &proof.DefaultHandler{},
 	}, nil
 }
 
@@ -556,4 +559,12 @@ func (s *Server) setupRoutes(e *echo.Echo) {
 	e.POST("/api/file/:id/prepare_to_pack", s.toEchoHandler(s.fileHandler.PrepareToPackFileHandler))
 	e.GET("/api/file/:id/retrieve", s.retrieveFile)
 	e.POST("/api/preparation/:id/source/:name/file", s.toEchoHandler(s.fileHandler.PushFileHandler))
+
+	// Proof
+	e.GET("/api/proof", s.toEchoHandler(func(ctx context.Context, db *gorm.DB, request proof.ListProofRequest) ([]model.Proof, error) {
+		return s.proofHandler.ListHandler(ctx, db, request)
+	}))
+	e.POST("/api/proof/sync", s.toEchoHandler(func(ctx context.Context, db *gorm.DB, request proof.SyncProofRequest) error {
+		return s.proofHandler.SyncHandler(ctx, db, s.lotusClient, request)
+	}))
 }

cmd/app.go

@@ -17,6 +17,7 @@ import (
 	"github.com/data-preservation-programs/singularity/cmd/deal/schedule"
 	"github.com/data-preservation-programs/singularity/cmd/dealtemplate"
 	"github.com/data-preservation-programs/singularity/cmd/ez"
+	"github.com/data-preservation-programs/singularity/cmd/proof"
 	"github.com/data-preservation-programs/singularity/cmd/run"
 	"github.com/data-preservation-programs/singularity/cmd/storage"
 	"github.com/data-preservation-programs/singularity/cmd/tool"
@@ -161,6 +162,15 @@ Upgrading:
 				dealtemplate.DeleteCmd,
 			},
 		},
+		{
+			Name:     "proof",
+			Usage:    "Proof management",
+			Category: "Operations",
+			Subcommands: []*cli.Command{
+				proof.ListCmd,
+				proof.SyncCmd,
+			},
+		},
 		{
 			Name:     "run",
 			Category: "Daemons",

cmd/proof/README.md

@@ -0,0 +1,213 @@
+# Proof CLI Commands
+
+The proof commands provide functionality to list and synchronize Filecoin proofs from the blockchain.
+
+## Commands
+
+### `singularity proof list`
+
+Lists proofs with optional filtering and pagination.
+
+#### Usage
+
+```bash
+singularity proof list [OPTIONS]
+```
+
+#### Options
+
+- `--deal-id <ID>` - Filter proofs by deal ID
+- `--proof-type <TYPE>` - Filter proofs by type: `replication`, `spacetime`
+- `--provider <PROVIDER>` - Filter proofs by storage provider (e.g., `f01000`)
+- `--verified` - Show only verified proofs
+- `--unverified` - Show only unverified proofs
+- `--limit <NUMBER>` - Limit number of results (default: 100)
+- `--offset <NUMBER>` - Offset for pagination (default: 0)
+
+#### Examples
+
+```bash
+# List all proofs
+singularity proof list
+
+# List proofs for a specific deal
+singularity proof list --deal-id 12345
+
+# List only replication proofs
+singularity proof list --proof-type replication
+
+# List only spacetime proofs
+singularity proof list --proof-type spacetime
+
+# List proofs from a specific provider
+singularity proof list --provider f01000
+
+# List only verified proofs
+singularity proof list --verified
+
+# List only unverified proofs
+singularity proof list --unverified
+
+# List with pagination
+singularity proof list --limit 50 --offset 100
+
+# Combined filters
+singularity proof list --provider f01000 --proof-type replication --verified
+```
+
+#### Output
+
+The command outputs a table with the following columns:
+- `ID` - Proof record ID
+- `DealID` - Associated deal ID
+- `ProofType` - Type of proof (replication/spacetime)
+- `MessageID` - Blockchain message CID
+- `Height` - Block height
+- `Method` - Proof method name
+- `Verified` - Whether proof was verified
+- `Provider` - Storage provider ID
+- `CreatedAt` - When proof was recorded
+
+With `--verbose` flag, additional columns are shown:
+- `BlockCID` - Block CID where proof was included
+- `SectorID` - Sector ID (if available)
+- `ErrorMsg` - Error message (if any)
+- `UpdatedAt` - Last update time
+
+### `singularity proof sync`
+
+Synchronizes proofs from the Filecoin blockchain into the local database.
+
+#### Usage
+
+```bash
+singularity proof sync [OPTIONS]
+```
+
+#### Options
+
+- `--deal-id <ID>` - Sync proofs for specific deal ID
+- `--provider <PROVIDER>` - Sync proofs for specific storage provider
+
+#### Examples
+
+```bash
+# Sync proofs for all active deals
+singularity proof sync
+
+# Sync proofs for a specific deal
+singularity proof sync --deal-id 12345
+
+# Sync proofs for a specific provider
+singularity proof sync --provider f01000
+
+# Sync proofs for specific provider and deal (both filters applied)
+singularity proof sync --provider f01000 --deal-id 12345
+```
+
+#### Behavior
+
+- If no options are provided, syncs proofs for all active deals
+- If `--deal-id` is provided, syncs proofs for that specific deal
+- If `--provider` is provided, syncs proofs for that specific provider
+- If both are provided, both filters are applied
+- The command looks back 2000 epochs (about 16 hours) for proof messages
+- Duplicate proofs are automatically skipped
+- Errors for individual messages are logged but don't stop the sync process
+
+#### Output
+
+The command outputs a success message upon completion:
+```json
+{
+  "status": "success"
+}
+```
+
+## Global Options
+
+These options can be used with any proof command:
+
+- `--json` - Output results in JSON format
+- `--verbose` - Show verbose output with additional details
+- `--database-connection-string <CONNECTION>` - Database connection string
+- `--lotus-api <URL>` - Lotus API endpoint (default: https://api.node.glif.io/rpc/v1)
+- `--lotus-token <TOKEN>` - Lotus API token
+
+## Examples
+
+### Basic Usage
+
+```bash
+# List first 10 proofs
+singularity proof list --limit 10
+
+# Sync proofs for all deals
+singularity proof sync
+
+# Check specific deal's proofs
+singularity proof list --deal-id 12345 --verbose
+```
+
+### JSON Output
+
+```bash
+# Get proof data in JSON format
+singularity --json proof list --deal-id 12345
+
+# Sync with JSON status
+singularity --json proof sync --provider f01000
+```
+
+### Filtering Examples
+
+```bash
+# Find all failed proofs
+singularity proof list --unverified --verbose
+
+# Check replication proofs for a provider
+singularity proof list --provider f01000 --proof-type replication
+
+# Paginate through large result sets
+singularity proof list --limit 100 --offset 0    # First 100
+singularity proof list --limit 100 --offset 100  # Next 100
+```
+
+### Monitoring Examples
+
+```bash
+# Monitor recent proofs
+singularity proof list --limit 20 --verbose
+
+# Sync and then check results
+singularity proof sync --provider f01000
+singularity proof list --provider f01000 --verbose
+```
+
+## Integration with Other Commands
+
+The proof commands work alongside other Singularity commands:
+
+```bash
+# List deals first, then check their proofs
+singularity deal list --provider f01000
+singularity proof list --provider f01000
+
+# Sync proofs after creating schedules
+singularity schedule create --provider f01000 ...
+singularity proof sync --provider f01000
+```
+
+## Error Handling
+
+- Database connection errors are displayed immediately
+- Lotus API errors during sync are logged but don't stop the process
+- Invalid command line arguments show usage help
+- Use `--verbose` flag to see detailed error information
+
+## Performance Notes
+
+- Listing proofs is fast due to database indexes
+- Syncing proofs may take time depending on the number of messages
+- Use specific filters (deal-id, provider) for faster sync operations
+- The sync process looks back 2000 epochs by default

cmd/proof/list.go

@@ -0,0 +1,92 @@
+package proof
+
+import (
+	"github.com/cockroachdb/errors"
+	"github.com/data-preservation-programs/singularity/cmd/cliutil"
+	"github.com/data-preservation-programs/singularity/database"
+	"github.com/data-preservation-programs/singularity/handler/proof"
+	"github.com/data-preservation-programs/singularity/model"
+	"github.com/urfave/cli/v2"
+)
+
+var ListCmd = &cli.Command{
+	Name:  "list",
+	Usage: "List proofs with optional filtering",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "deal-id",
+			Usage: "Filter proofs by deal ID",
+		},
+		&cli.StringFlag{
+			Name:  "proof-type",
+			Usage: "Filter proofs by type: replication, spacetime",
+		},
+		&cli.StringFlag{
+			Name:  "provider",
+			Usage: "Filter proofs by storage provider",
+		},
+		&cli.BoolFlag{
+			Name:  "verified",
+			Usage: "Filter proofs by verification status",
+		},
+		&cli.BoolFlag{
+			Name:  "unverified",
+			Usage: "Filter proofs by unverified status",
+		},
+		&cli.IntFlag{
+			Name:  "limit",
+			Usage: "Limit number of results",
+			Value: 100,
+		},
+		&cli.IntFlag{
+			Name:  "offset",
+			Usage: "Offset for pagination",
+			Value: 0,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		db, closer, err := database.OpenFromCLI(c)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		defer func() { _ = closer.Close() }()
+
+		request := proof.ListProofRequest{
+			Limit:  c.Int("limit"),
+			Offset: c.Int("offset"),
+		}
+
+		if c.IsSet("deal-id") {
+			dealID := c.Uint64("deal-id")
+			request.DealID = &dealID
+		}
+
+		if c.IsSet("proof-type") {
+			proofType := model.ProofType(c.String("proof-type"))
+			request.ProofType = &proofType
+		}
+
+		if c.IsSet("provider") {
+			provider := c.String("provider")
+			request.Provider = &provider
+		}
+
+		if c.IsSet("verified") {
+			verified := true
+			request.Verified = &verified
+		}
+
+		if c.IsSet("unverified") {
+			verified := false
+			request.Verified = &verified
+		}
+
+		proofs, err := proof.Default.ListHandler(c.Context, db, request)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		cliutil.Print(c, proofs)
+		return nil
+	},
+}