diff --git a/api/api.go b/api/api.go index bcb59fd0..06eb6369 100644 --- a/api/api.go +++ b/api/api.go @@ -22,6 +22,7 @@ import ( "github.com/data-preservation-programs/singularity/handler/file" "github.com/data-preservation-programs/singularity/handler/handlererror" "github.com/data-preservation-programs/singularity/handler/job" + "github.com/data-preservation-programs/singularity/handler/proof" "github.com/data-preservation-programs/singularity/handler/storage" "github.com/data-preservation-programs/singularity/handler/wallet" "github.com/data-preservation-programs/singularity/model" @@ -58,6 +59,7 @@ type Server struct { fileHandler file.Handler jobHandler job.Handler scheduleHandler schedule.Handler + proofHandler proof.Handler } func Run(c *cli.Context) error { @@ -138,6 +140,7 @@ func InitServer(ctx context.Context, params APIParams) (*Server, error) { fileHandler: &file.DefaultHandler{}, jobHandler: &job.DefaultHandler{}, scheduleHandler: &schedule.DefaultHandler{}, + proofHandler: &proof.DefaultHandler{}, }, nil } @@ -556,4 +559,12 @@ func (s *Server) setupRoutes(e *echo.Echo) { e.POST("/api/file/:id/prepare_to_pack", s.toEchoHandler(s.fileHandler.PrepareToPackFileHandler)) e.GET("/api/file/:id/retrieve", s.retrieveFile) e.POST("/api/preparation/:id/source/:name/file", s.toEchoHandler(s.fileHandler.PushFileHandler)) + + // Proof + e.GET("/api/proof", s.toEchoHandler(func(ctx context.Context, db *gorm.DB, request proof.ListProofRequest) ([]model.Proof, error) { + return s.proofHandler.ListHandler(ctx, db, request) + })) + e.POST("/api/proof/sync", s.toEchoHandler(func(ctx context.Context, db *gorm.DB, request proof.SyncProofRequest) error { + return s.proofHandler.SyncHandler(ctx, db, s.lotusClient, request) + })) } diff --git a/cmd/app.go b/cmd/app.go index 6955de96..d876bbf5 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -17,6 +17,7 @@ import ( "github.com/data-preservation-programs/singularity/cmd/deal/schedule" "github.com/data-preservation-programs/singularity/cmd/dealtemplate" "github.com/data-preservation-programs/singularity/cmd/ez" + "github.com/data-preservation-programs/singularity/cmd/proof" "github.com/data-preservation-programs/singularity/cmd/run" "github.com/data-preservation-programs/singularity/cmd/storage" "github.com/data-preservation-programs/singularity/cmd/tool" @@ -161,6 +162,15 @@ Upgrading: dealtemplate.DeleteCmd, }, }, + { + Name: "proof", + Usage: "Proof management", + Category: "Operations", + Subcommands: []*cli.Command{ + proof.ListCmd, + proof.SyncCmd, + }, + }, { Name: "run", Category: "Daemons", diff --git a/cmd/proof/README.md b/cmd/proof/README.md new file mode 100644 index 00000000..1bca64e3 --- /dev/null +++ b/cmd/proof/README.md @@ -0,0 +1,213 @@ +# Proof CLI Commands + +The proof commands provide functionality to list and synchronize Filecoin proofs from the blockchain. + +## Commands + +### `singularity proof list` + +Lists proofs with optional filtering and pagination. + +#### Usage + +```bash +singularity proof list [OPTIONS] +``` + +#### Options + +- `--deal-id ` - Filter proofs by deal ID +- `--proof-type ` - Filter proofs by type: `replication`, `spacetime` +- `--provider ` - Filter proofs by storage provider (e.g., `f01000`) +- `--verified` - Show only verified proofs +- `--unverified` - Show only unverified proofs +- `--limit ` - Limit number of results (default: 100) +- `--offset ` - Offset for pagination (default: 0) + +#### Examples + +```bash +# List all proofs +singularity proof list + +# List proofs for a specific deal +singularity proof list --deal-id 12345 + +# List only replication proofs +singularity proof list --proof-type replication + +# List only spacetime proofs +singularity proof list --proof-type spacetime + +# List proofs from a specific provider +singularity proof list --provider f01000 + +# List only verified proofs +singularity proof list --verified + +# List only unverified proofs +singularity proof list --unverified + +# List with pagination +singularity proof list --limit 50 --offset 100 + +# Combined filters +singularity proof list --provider f01000 --proof-type replication --verified +``` + +#### Output + +The command outputs a table with the following columns: +- `ID` - Proof record ID +- `DealID` - Associated deal ID +- `ProofType` - Type of proof (replication/spacetime) +- `MessageID` - Blockchain message CID +- `Height` - Block height +- `Method` - Proof method name +- `Verified` - Whether proof was verified +- `Provider` - Storage provider ID +- `CreatedAt` - When proof was recorded + +With `--verbose` flag, additional columns are shown: +- `BlockCID` - Block CID where proof was included +- `SectorID` - Sector ID (if available) +- `ErrorMsg` - Error message (if any) +- `UpdatedAt` - Last update time + +### `singularity proof sync` + +Synchronizes proofs from the Filecoin blockchain into the local database. + +#### Usage + +```bash +singularity proof sync [OPTIONS] +``` + +#### Options + +- `--deal-id ` - Sync proofs for specific deal ID +- `--provider ` - Sync proofs for specific storage provider + +#### Examples + +```bash +# Sync proofs for all active deals +singularity proof sync + +# Sync proofs for a specific deal +singularity proof sync --deal-id 12345 + +# Sync proofs for a specific provider +singularity proof sync --provider f01000 + +# Sync proofs for specific provider and deal (both filters applied) +singularity proof sync --provider f01000 --deal-id 12345 +``` + +#### Behavior + +- If no options are provided, syncs proofs for all active deals +- If `--deal-id` is provided, syncs proofs for that specific deal +- If `--provider` is provided, syncs proofs for that specific provider +- If both are provided, both filters are applied +- The command looks back 2000 epochs (about 16 hours) for proof messages +- Duplicate proofs are automatically skipped +- Errors for individual messages are logged but don't stop the sync process + +#### Output + +The command outputs a success message upon completion: +```json +{ + "status": "success" +} +``` + +## Global Options + +These options can be used with any proof command: + +- `--json` - Output results in JSON format +- `--verbose` - Show verbose output with additional details +- `--database-connection-string ` - Database connection string +- `--lotus-api ` - Lotus API endpoint (default: https://api.node.glif.io/rpc/v1) +- `--lotus-token ` - Lotus API token + +## Examples + +### Basic Usage + +```bash +# List first 10 proofs +singularity proof list --limit 10 + +# Sync proofs for all deals +singularity proof sync + +# Check specific deal's proofs +singularity proof list --deal-id 12345 --verbose +``` + +### JSON Output + +```bash +# Get proof data in JSON format +singularity --json proof list --deal-id 12345 + +# Sync with JSON status +singularity --json proof sync --provider f01000 +``` + +### Filtering Examples + +```bash +# Find all failed proofs +singularity proof list --unverified --verbose + +# Check replication proofs for a provider +singularity proof list --provider f01000 --proof-type replication + +# Paginate through large result sets +singularity proof list --limit 100 --offset 0 # First 100 +singularity proof list --limit 100 --offset 100 # Next 100 +``` + +### Monitoring Examples + +```bash +# Monitor recent proofs +singularity proof list --limit 20 --verbose + +# Sync and then check results +singularity proof sync --provider f01000 +singularity proof list --provider f01000 --verbose +``` + +## Integration with Other Commands + +The proof commands work alongside other Singularity commands: + +```bash +# List deals first, then check their proofs +singularity deal list --provider f01000 +singularity proof list --provider f01000 + +# Sync proofs after creating schedules +singularity schedule create --provider f01000 ... +singularity proof sync --provider f01000 +``` + +## Error Handling + +- Database connection errors are displayed immediately +- Lotus API errors during sync are logged but don't stop the process +- Invalid command line arguments show usage help +- Use `--verbose` flag to see detailed error information + +## Performance Notes + +- Listing proofs is fast due to database indexes +- Syncing proofs may take time depending on the number of messages +- Use specific filters (deal-id, provider) for faster sync operations +- The sync process looks back 2000 epochs by default \ No newline at end of file diff --git a/cmd/proof/list.go b/cmd/proof/list.go new file mode 100644 index 00000000..8678cd20 --- /dev/null +++ b/cmd/proof/list.go @@ -0,0 +1,92 @@ +package proof + +import ( + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/cmd/cliutil" + "github.com/data-preservation-programs/singularity/database" + "github.com/data-preservation-programs/singularity/handler/proof" + "github.com/data-preservation-programs/singularity/model" + "github.com/urfave/cli/v2" +) + +var ListCmd = &cli.Command{ + Name: "list", + Usage: "List proofs with optional filtering", + Flags: []cli.Flag{ + &cli.Uint64Flag{ + Name: "deal-id", + Usage: "Filter proofs by deal ID", + }, + &cli.StringFlag{ + Name: "proof-type", + Usage: "Filter proofs by type: replication, spacetime", + }, + &cli.StringFlag{ + Name: "provider", + Usage: "Filter proofs by storage provider", + }, + &cli.BoolFlag{ + Name: "verified", + Usage: "Filter proofs by verification status", + }, + &cli.BoolFlag{ + Name: "unverified", + Usage: "Filter proofs by unverified status", + }, + &cli.IntFlag{ + Name: "limit", + Usage: "Limit number of results", + Value: 100, + }, + &cli.IntFlag{ + Name: "offset", + Usage: "Offset for pagination", + Value: 0, + }, + }, + Action: func(c *cli.Context) error { + db, closer, err := database.OpenFromCLI(c) + if err != nil { + return errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + + request := proof.ListProofRequest{ + Limit: c.Int("limit"), + Offset: c.Int("offset"), + } + + if c.IsSet("deal-id") { + dealID := c.Uint64("deal-id") + request.DealID = &dealID + } + + if c.IsSet("proof-type") { + proofType := model.ProofType(c.String("proof-type")) + request.ProofType = &proofType + } + + if c.IsSet("provider") { + provider := c.String("provider") + request.Provider = &provider + } + + if c.IsSet("verified") { + verified := true + request.Verified = &verified + } + + if c.IsSet("unverified") { + verified := false + request.Verified = &verified + } + + proofs, err := proof.Default.ListHandler(c.Context, db, request) + if err != nil { + return errors.WithStack(err) + } + + cliutil.Print(c, proofs) + return nil + }, +} diff --git a/cmd/proof/sync.go b/cmd/proof/sync.go new file mode 100644 index 00000000..c53dfa66 --- /dev/null +++ b/cmd/proof/sync.go @@ -0,0 +1,54 @@ +package proof + +import ( + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/cmd/cliutil" + "github.com/data-preservation-programs/singularity/database" + "github.com/data-preservation-programs/singularity/handler/proof" + "github.com/data-preservation-programs/singularity/util" + "github.com/urfave/cli/v2" +) + +var SyncCmd = &cli.Command{ + Name: "sync", + Usage: "Sync proofs from Filecoin blockchain", + Flags: []cli.Flag{ + &cli.Uint64Flag{ + Name: "deal-id", + Usage: "Sync proofs for specific deal ID", + }, + &cli.StringFlag{ + Name: "provider", + Usage: "Sync proofs for specific storage provider", + }, + }, + Action: func(c *cli.Context) error { + db, closer, err := database.OpenFromCLI(c) + if err != nil { + return errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + + lotusClient := util.NewLotusClient(c.String("lotus-api"), c.String("lotus-token")) + + request := proof.SyncProofRequest{} + + if c.IsSet("deal-id") { + dealID := c.Uint64("deal-id") + request.DealID = &dealID + } + + if c.IsSet("provider") { + provider := c.String("provider") + request.Provider = &provider + } + + err = proof.Default.SyncHandler(c.Context, db, lotusClient, request) + if err != nil { + return errors.WithStack(err) + } + + cliutil.Print(c, map[string]string{"status": "success"}) + return nil + }, +} diff --git a/cmd/proof_test.go b/cmd/proof_test.go new file mode 100644 index 00000000..04ba6a88 --- /dev/null +++ b/cmd/proof_test.go @@ -0,0 +1,218 @@ +package cmd + +import ( + "context" + "testing" + "time" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/handler/proof" + "github.com/data-preservation-programs/singularity/model" + "github.com/data-preservation-programs/singularity/util/testutil" + "github.com/gotidy/ptr" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func swapProofHandler(mockHandler proof.Handler) func() { + actual := proof.Default + proof.Default = mockHandler + return func() { + proof.Default = actual + } +} + +func TestProofListHandler(t *testing.T) { + testutil.OneWithoutReset(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + runner := NewRunner() + defer runner.Save(t) + mockHandler := new(proof.MockProof) + defer swapProofHandler(mockHandler)() + + mockHandler.On("ListHandler", mock.Anything, mock.Anything, mock.Anything).Return([]model.Proof{ + { + ID: 1, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DealID: ptr.Of(uint64(100)), + ProofType: model.ProofOfReplication, + MessageID: "bafy2bzacea1", + BlockCID: "bafy2bzaceb1", + Height: 1000000, + Method: "ProveCommitSector", + Verified: true, + SectorID: ptr.Of(uint64(456)), + Provider: "f01000", + ErrorMsg: "", + }, + { + ID: 2, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DealID: ptr.Of(uint64(200)), + ProofType: model.ProofOfSpacetime, + MessageID: "bafy2bzacea2", + BlockCID: "bafy2bzaceb2", + Height: 2000000, + Method: "SubmitWindowedPoSt", + Verified: false, + SectorID: ptr.Of(uint64(789)), + Provider: "f01001", + ErrorMsg: "validation failed", + }, + }, nil) + + // Test basic list command + _, _, err := runner.Run(ctx, "singularity proof list") + require.NoError(t, err) + + // Test with verbose output + _, _, err = runner.Run(ctx, "singularity --verbose proof list") + require.NoError(t, err) + + // Test with JSON output + _, _, err = runner.Run(ctx, "singularity --json proof list") + require.NoError(t, err) + + // Test with filters + _, _, err = runner.Run(ctx, "singularity proof list --deal-id 100") + require.NoError(t, err) + + _, _, err = runner.Run(ctx, "singularity proof list --proof-type replication") + require.NoError(t, err) + + _, _, err = runner.Run(ctx, "singularity proof list --provider f01000") + require.NoError(t, err) + + _, _, err = runner.Run(ctx, "singularity proof list --verified") + require.NoError(t, err) + + _, _, err = runner.Run(ctx, "singularity proof list --unverified") + require.NoError(t, err) + + // Test with pagination + _, _, err = runner.Run(ctx, "singularity proof list --limit 10 --offset 5") + require.NoError(t, err) + + // Test with multiple filters + _, _, err = runner.Run(ctx, "singularity proof list --deal-id 100 --proof-type replication --verified") + require.NoError(t, err) + }) +} + +func TestProofSyncHandler(t *testing.T) { + testutil.OneWithoutReset(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + runner := NewRunner() + defer runner.Save(t) + mockHandler := new(proof.MockProof) + defer swapProofHandler(mockHandler)() + + mockHandler.On("SyncHandler", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + // Test basic sync command + _, _, err := runner.Run(ctx, "singularity proof sync") + require.NoError(t, err) + + // Test sync for specific deal + _, _, err = runner.Run(ctx, "singularity proof sync --deal-id 123") + require.NoError(t, err) + + // Test sync for specific provider + _, _, err = runner.Run(ctx, "singularity proof sync --provider f01000") + require.NoError(t, err) + + // Test with verbose output + _, _, err = runner.Run(ctx, "singularity --verbose proof sync") + require.NoError(t, err) + + // Test with JSON output + _, _, err = runner.Run(ctx, "singularity --json proof sync") + require.NoError(t, err) + + // Test with both deal-id and provider (should work) + _, _, err = runner.Run(ctx, "singularity proof sync --deal-id 123 --provider f01000") + require.NoError(t, err) + }) +} + +func TestProofCommandHelp(t *testing.T) { + testutil.OneWithoutReset(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + runner := NewRunner() + defer runner.Save(t) + + // Test help commands + _, _, err := runner.Run(ctx, "singularity proof --help") + require.NoError(t, err) + + _, _, err = runner.Run(ctx, "singularity proof list --help") + require.NoError(t, err) + + _, _, err = runner.Run(ctx, "singularity proof sync --help") + require.NoError(t, err) + }) +} + +func TestProofListHandlerWithErrors(t *testing.T) { + testutil.OneWithoutReset(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + runner := NewRunner() + defer runner.Save(t) + mockHandler := new(proof.MockProof) + defer swapProofHandler(mockHandler)() + + // Mock an error response + mockHandler.On("ListHandler", mock.Anything, mock.Anything, mock.Anything).Return([]model.Proof{}, + errors.New("database connection failed")) + + // Test that error is properly handled + _, _, err := runner.Run(ctx, "singularity proof list") + require.Error(t, err) + require.Contains(t, err.Error(), "database connection failed") + }) +} + +func TestProofSyncHandlerWithErrors(t *testing.T) { + testutil.OneWithoutReset(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + runner := NewRunner() + defer runner.Save(t) + mockHandler := new(proof.MockProof) + defer swapProofHandler(mockHandler)() + + // Mock an error response + mockHandler.On("SyncHandler", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return( + errors.New("lotus client connection failed")) + + // Test that error is properly handled + _, _, err := runner.Run(ctx, "singularity proof sync") + require.Error(t, err) + require.Contains(t, err.Error(), "lotus client connection failed") + }) +} + +func TestProofCommandValidation(t *testing.T) { + testutil.OneWithoutReset(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + runner := NewRunner() + defer runner.Save(t) + + // Test invalid proof type + _, _, err := runner.Run(ctx, "singularity proof list --proof-type invalid") + // This should not error at CLI level since validation happens in handler + require.NoError(t, err) + + // Test invalid deal ID format - this should be caught by CLI flag parsing + _, _, err = runner.Run(ctx, "singularity proof list --deal-id invalid") + require.Error(t, err) + + // Test invalid limit format + _, _, err = runner.Run(ctx, "singularity proof list --limit invalid") + require.Error(t, err) + + // Test invalid offset format + _, _, err = runner.Run(ctx, "singularity proof list --offset invalid") + require.Error(t, err) + + // Test invalid provider format for sync + _, _, err = runner.Run(ctx, "singularity proof sync --deal-id invalid") + require.Error(t, err) + }) +} diff --git a/handler/proof/interface.go b/handler/proof/interface.go new file mode 100644 index 00000000..4df052a6 --- /dev/null +++ b/handler/proof/interface.go @@ -0,0 +1,35 @@ +package proof + +import ( + "context" + + "github.com/data-preservation-programs/singularity/model" + "github.com/stretchr/testify/mock" + "github.com/ybbus/jsonrpc/v3" + "gorm.io/gorm" +) + +type Handler interface { + ListHandler(ctx context.Context, db *gorm.DB, request ListProofRequest) ([]model.Proof, error) + SyncHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, request SyncProofRequest) error +} + +type DefaultHandler struct{} + +var Default Handler = &DefaultHandler{} + +var _ Handler = &MockProof{} + +type MockProof struct { + mock.Mock +} + +func (m *MockProof) ListHandler(ctx context.Context, db *gorm.DB, request ListProofRequest) ([]model.Proof, error) { + args := m.Called(ctx, db, request) + return args.Get(0).([]model.Proof), args.Error(1) +} + +func (m *MockProof) SyncHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, request SyncProofRequest) error { + args := m.Called(ctx, db, lotusClient, request) + return args.Error(0) +} diff --git a/handler/proof/list.go b/handler/proof/list.go new file mode 100644 index 00000000..0c111614 --- /dev/null +++ b/handler/proof/list.go @@ -0,0 +1,88 @@ +package proof + +import ( + "context" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/model" + "gorm.io/gorm" +) + +type ListProofRequest struct { + DealID *uint64 `json:"dealId"` // deal ID filter + ProofType *model.ProofType `json:"proofType"` // proof type filter + Provider *string `json:"provider"` // provider filter + Verified *bool `json:"verified"` // verified status filter + Limit int `json:"limit"` // limit number of results + Offset int `json:"offset"` // offset for pagination +} + +// ListHandler retrieves a list of proofs from the database based on the specified filtering criteria in ListProofRequest. +// +// The function takes advantage of the conditional nature of the ListProofRequest to construct the final query. It +// filters proofs based on various conditions such as deal ID, proof type, provider, and verification status +// as specified in the request. +// +// The function begins by associating the provided context with the database connection. It then successively builds +// upon a GORM statement by appending where clauses based on the parameters in the request. +// +// Parameters: +// - ctx: The context for the operation which provides facilities for timeouts and cancellations. +// - db: The database connection for performing CRUD operations related to proofs. +// - request: The request object which contains the filtering criteria for the proofs retrieval. +// +// Returns: +// - A slice of model.Proof objects matching the filtering criteria. +// - An error indicating any issues that occurred during the database operation. +func (DefaultHandler) ListHandler(ctx context.Context, db *gorm.DB, request ListProofRequest) ([]model.Proof, error) { + db = db.WithContext(ctx) + + // Set default limit if not provided + if request.Limit == 0 { + request.Limit = 100 + } + + statement := db + + if request.DealID != nil { + statement = statement.Where("deal_id = ?", *request.DealID) + } + + if request.ProofType != nil { + statement = statement.Where("proof_type = ?", *request.ProofType) + } + + if request.Provider != nil { + statement = statement.Where("provider = ?", *request.Provider) + } + + if request.Verified != nil { + statement = statement.Where("verified = ?", *request.Verified) + } + + var proofs []model.Proof + err := statement.Preload("Deal"). + Limit(request.Limit). + Offset(request.Offset). + Order("created_at DESC"). + Find(&proofs).Error + + if err != nil { + return nil, errors.WithStack(err) + } + + return proofs, nil +} + +// @ID ListProofs +// @Summary List all proofs +// @Description List all proofs with optional filtering +// @Tags Proof +// @Accept json +// @Produce json +// @Param request body ListProofRequest true "ListProofRequest" +// @Success 200 {array} model.Proof +// @Failure 400 {object} api.HTTPError +// @Failure 500 {object} api.HTTPError +// @Router /proof [get] +func _() {} diff --git a/handler/proof/list_test.go b/handler/proof/list_test.go new file mode 100644 index 00000000..a6c8d7ee --- /dev/null +++ b/handler/proof/list_test.go @@ -0,0 +1,202 @@ +package proof + +import ( + "context" + "testing" + + "github.com/data-preservation-programs/singularity/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func setupTestDB(t *testing.T) *gorm.DB { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + + err = db.AutoMigrate(&model.Deal{}, &model.Proof{}) + require.NoError(t, err) + + return db +} + +func TestDefaultHandler_ListHandler(t *testing.T) { + db := setupTestDB(t) + handler := &DefaultHandler{} + ctx := context.Background() + + // Create test data + deal1 := model.Deal{ + DealID: uint64Ptr(1), + Provider: "f01000", + State: model.DealActive, + } + deal2 := model.Deal{ + DealID: uint64Ptr(2), + Provider: "f01001", + State: model.DealActive, + } + require.NoError(t, db.Create(&deal1).Error) + require.NoError(t, db.Create(&deal2).Error) + + proof1 := model.Proof{ + DealID: uint64Ptr(1), + ProofType: model.ProofOfReplication, + Provider: "f01000", + MessageID: "bafy2bzacea1", + Height: 1000, + Method: "ProveCommitSector", + Verified: true, + } + proof2 := model.Proof{ + DealID: uint64Ptr(2), + ProofType: model.ProofOfSpacetime, + Provider: "f01001", + MessageID: "bafy2bzacea2", + Height: 2000, + Method: "SubmitWindowedPoSt", + Verified: false, + } + proof3 := model.Proof{ + ProofType: model.ProofOfReplication, + Provider: "f01000", + MessageID: "bafy2bzacea3", + Height: 3000, + Method: "PreCommitSector", + Verified: true, + } + require.NoError(t, db.Create(&proof1).Error) + require.NoError(t, db.Create(&proof2).Error) + require.NoError(t, db.Create(&proof3).Error) + + tests := []struct { + name string + request ListProofRequest + expected int + }{ + { + name: "list all proofs", + request: ListProofRequest{}, + expected: 3, + }, + { + name: "filter by deal ID", + request: ListProofRequest{ + DealID: uint64Ptr(1), + }, + expected: 1, + }, + { + name: "filter by proof type", + request: ListProofRequest{ + ProofType: proofTypePtr(model.ProofOfReplication), + }, + expected: 2, + }, + { + name: "filter by provider", + request: ListProofRequest{ + Provider: stringPtr("f01000"), + }, + expected: 2, + }, + { + name: "filter by verified status", + request: ListProofRequest{ + Verified: boolPtr(true), + }, + expected: 2, + }, + { + name: "filter by verified status false", + request: ListProofRequest{ + Verified: boolPtr(false), + }, + expected: 1, + }, + { + name: "filter with limit", + request: ListProofRequest{ + Limit: 2, + }, + expected: 2, + }, + { + name: "filter with offset", + request: ListProofRequest{ + Limit: 10, + Offset: 1, + }, + expected: 2, + }, + { + name: "multiple filters", + request: ListProofRequest{ + Provider: stringPtr("f01000"), + ProofType: proofTypePtr(model.ProofOfReplication), + Verified: boolPtr(true), + }, + expected: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proofs, err := handler.ListHandler(ctx, db, tt.request) + require.NoError(t, err) + assert.Len(t, proofs, tt.expected) + + // Verify proofs are ordered by created_at DESC + if len(proofs) > 1 { + for i := 1; i < len(proofs); i++ { + assert.True(t, proofs[i-1].CreatedAt.After(proofs[i].CreatedAt) || proofs[i-1].CreatedAt.Equal(proofs[i].CreatedAt)) + } + } + }) + } +} + +func TestDefaultHandler_ListHandler_DefaultLimit(t *testing.T) { + db := setupTestDB(t) + handler := &DefaultHandler{} + ctx := context.Background() + + // Test default limit is applied when limit is 0 + request := ListProofRequest{Limit: 0} + _, err := handler.ListHandler(ctx, db, request) + require.NoError(t, err) + + // The limit should be set to 100 internally + // We can't directly test this without exposing internal state, + // but we can verify the query doesn't fail +} + +func TestDefaultHandler_ListHandler_EmptyResult(t *testing.T) { + db := setupTestDB(t) + handler := &DefaultHandler{} + ctx := context.Background() + + // Test with no proofs in database + request := ListProofRequest{} + proofs, err := handler.ListHandler(ctx, db, request) + require.NoError(t, err) + assert.Empty(t, proofs) +} + +// Helper functions +func uint64Ptr(v uint64) *uint64 { + return &v +} + +func stringPtr(v string) *string { + return &v +} + +func boolPtr(v bool) *bool { + return &v +} + +func proofTypePtr(v model.ProofType) *model.ProofType { + return &v +} diff --git a/handler/proof/sync.go b/handler/proof/sync.go new file mode 100644 index 00000000..ab9a551e --- /dev/null +++ b/handler/proof/sync.go @@ -0,0 +1,286 @@ +package proof + +import ( + "context" + "fmt" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/model" + "github.com/ybbus/jsonrpc/v3" + "gorm.io/gorm" +) + +type SyncProofRequest struct { + DealID *uint64 `json:"dealId"` // specific deal ID to sync proofs for + Provider *string `json:"provider"` // specific provider to sync proofs for +} + +// Message represents a Filecoin message from the chain +type Message struct { + To string `json:"To"` + From string `json:"From"` + Method uint64 `json:"Method"` + Params string `json:"Params"` +} + +// MessageReceipt represents a message receipt +type MessageReceipt struct { + ExitCode int `json:"ExitCode"` + Return string `json:"Return"` + GasUsed int64 `json:"GasUsed"` +} + +// InvocResult represents the result of StateReplay +type InvocResult struct { + MsgCid struct { + CID string `json:"/"` + } `json:"MsgCid"` + Msg Message `json:"Msg"` + MsgRct MessageReceipt `json:"MsgRct"` + Error string `json:"Error"` + Duration int64 `json:"Duration"` +} + +// TipSet represents a tipset with block CIDs +type TipSet []struct { + CID string `json:"/"` +} + +// ChainHead represents the chain head response +type ChainHead struct { + Height int64 `json:"Height"` + Cids TipSet `json:"Cids"` +} + +// Miner actor method numbers for proof-related operations +const ( + MethodSubmitWindowedPoSt = 5 // WindowPoSt proofs (spacetime) + MethodProveCommitSector = 7 // Sector commit proofs (replication) + MethodPreCommitSector = 6 // PreCommit sector +) + +// SyncHandler synchronizes proofs from the Filecoin chain into the database. +// +// This handler can sync proofs for all deals, a specific deal, or a specific provider +// based on the request parameters. It fetches messages from the Filecoin chain, +// identifies proof-related messages, and stores them in the database. +// +// Parameters: +// - ctx: The context for the operation which provides facilities for timeouts and cancellations. +// - db: The database connection for performing CRUD operations related to proofs. +// - lotusClient: The Lotus client for interacting with the Filecoin chain. +// - request: The request object which contains the sync criteria. +// +// Returns: +// - An error indicating any issues that occurred during the sync operation. +func (DefaultHandler) SyncHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, request SyncProofRequest) error { + db = db.WithContext(ctx) + + if request.DealID != nil { + // Sync proofs for a specific deal + return syncProofsForDeal(ctx, db, lotusClient, *request.DealID) + } + + if request.Provider != nil { + // Sync proofs for a specific provider + return syncProofsForProvider(ctx, db, lotusClient, *request.Provider, nil) + } + + // Sync proofs for all active deals + return syncAllProofs(ctx, db, lotusClient) +} + +func syncProofsForDeal(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, dealID uint64) error { + // Get the deal from database + var deal model.Deal + err := db.Where("deal_id = ?", dealID).First(&deal).Error + if err != nil { + return errors.Wrapf(err, "failed to find deal with ID %d", dealID) + } + + // Search for messages related to this deal's provider + return syncProofsForProvider(ctx, db, lotusClient, deal.Provider, &dealID) +} + +func syncAllProofs(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient) error { + // Get all active deals + var deals []model.Deal + err := db.Where("state IN ?", []string{ + string(model.DealActive), + string(model.DealPublished), + }).Find(&deals).Error + if err != nil { + return errors.WithStack(err) + } + + // Group deals by provider to avoid duplicate work + providerDeals := make(map[string][]uint64) + for _, deal := range deals { + if deal.DealID != nil { + providerDeals[deal.Provider] = append(providerDeals[deal.Provider], *deal.DealID) + } + } + + // Sync proofs for each provider + for provider := range providerDeals { + if err := syncProofsForProvider(ctx, db, lotusClient, provider, nil); err != nil { + // Log error but continue with other providers + fmt.Printf("Error syncing proofs for provider %s: %v\n", provider, err) + } + } + + return nil +} + +func syncProofsForProvider(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, provider string, dealID *uint64) error { + // Get current chain head + var chainHead ChainHead + err := lotusClient.CallFor(ctx, &chainHead, "Filecoin.ChainHead") + if err != nil { + return errors.Wrap(err, "failed to get chain head") + } + + // Look back 2000 epochs (about 16 hours) for proof messages + lookbackEpochs := int64(2000) + fromHeight := chainHead.Height - lookbackEpochs + if fromHeight < 0 { + fromHeight = 0 + } + + // Search for messages from this provider + messageFilter := map[string]interface{}{ + "From": provider, + } + + var messageCids []struct { + CID string `json:"/"` + } + err = lotusClient.CallFor(ctx, &messageCids, "Filecoin.StateListMessages", + messageFilter, chainHead.Cids, fromHeight) + if err != nil { + return errors.Wrap(err, "failed to list messages") + } + + // Process each message + for _, msgCid := range messageCids { + if err := processMessage(ctx, db, lotusClient, msgCid.CID, provider, dealID); err != nil { + // Log error but continue processing other messages + fmt.Printf("Error processing message %s: %v\n", msgCid.CID, err) + } + } + + return nil +} + +func processMessage(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, messageCid, provider string, dealID *uint64) error { + // Get message details + var msg Message + err := lotusClient.CallFor(ctx, &msg, "Filecoin.ChainGetMessage", map[string]string{ + "/": messageCid, + }) + if err != nil { + return errors.Wrap(err, "failed to get message") + } + + // Check if this is a proof-related message + var proofType model.ProofType + var method string + + switch msg.Method { + case MethodSubmitWindowedPoSt: + proofType = model.ProofOfSpacetime + method = "SubmitWindowedPoSt" + case MethodProveCommitSector: + proofType = model.ProofOfReplication + method = "ProveCommitSector" + case MethodPreCommitSector: + proofType = model.ProofOfReplication + method = "PreCommitSector" + default: + // Not a proof message, skip + return nil + } + + // Get message execution result + var invocResult InvocResult + err = lotusClient.CallFor(ctx, &invocResult, "Filecoin.StateReplay", + nil, map[string]string{"/": messageCid}) + if err != nil { + return errors.Wrap(err, "failed to replay message") + } + + // Search for the message to get the tipset and height + var msgLookup struct { + Message struct { + CID string `json:"/"` + } `json:"Message"` + Receipt MessageReceipt `json:"Receipt"` + TipSet TipSet `json:"TipSet"` + Height int64 `json:"Height"` + } + + err = lotusClient.CallFor(ctx, &msgLookup, "Filecoin.StateSearchMsg", + nil, map[string]string{"/": messageCid}, 2000, true) + if err != nil { + return errors.Wrap(err, "failed to search message") + } + + // Extract sector ID from message params if possible + var sectorID *uint64 + if len(msg.Params) > 0 { + // This is a simplified parsing - in practice you'd need to decode the CBOR params + // For now, we'll leave it as nil + } + + // Create block CID from tipset + var blockCID string + if len(msgLookup.TipSet) > 0 { + blockCID = msgLookup.TipSet[0].CID + } + + // Check if proof already exists + var existingProof model.Proof + err = db.Where("message_id = ?", messageCid).First(&existingProof).Error + if err == nil { + // Proof already exists, skip + return nil + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return errors.Wrap(err, "failed to check existing proof") + } + + // Create new proof record + proof := model.Proof{ + DealID: dealID, + ProofType: proofType, + MessageID: messageCid, + BlockCID: blockCID, + Height: msgLookup.Height, + Method: method, + Verified: invocResult.MsgRct.ExitCode == 0, + SectorID: sectorID, + Provider: provider, + ErrorMsg: invocResult.Error, + } + + // Save proof to database + err = db.Create(&proof).Error + if err != nil { + return errors.Wrap(err, "failed to create proof record") + } + + return nil +} + +// @ID SyncProofs +// @Summary Sync proofs from Filecoin chain +// @Description Synchronize proofs from the Filecoin chain into the database +// @Tags Proof +// @Accept json +// @Produce json +// @Param request body SyncProofRequest true "SyncProofRequest" +// @Success 200 {object} string "success" +// @Failure 400 {object} api.HTTPError +// @Failure 500 {object} api.HTTPError +// @Router /proof/sync [post] +func _() {} diff --git a/migrate/migrations/202507091738_create_proofs.go b/migrate/migrations/202507091738_create_proofs.go new file mode 100644 index 00000000..4bb6ea8a --- /dev/null +++ b/migrate/migrations/202507091738_create_proofs.go @@ -0,0 +1,52 @@ +package migrations + +import ( + "time" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// ProofType represents the type of proof +type ProofType string + +const ( + ProofOfReplication ProofType = "replication" + ProofOfSpacetime ProofType = "spacetime" +) + +// Proof represents a proof record at the time of migration +type Proof struct { + ID uint64 `gorm:"primaryKey" json:"id"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + + // Link to existing deal + DealID *uint64 `gorm:"index" json:"dealId"` + + // Proof details + ProofType ProofType `gorm:"index" json:"proofType"` + MessageID string `gorm:"index" json:"messageId"` // Message CID + BlockCID string `gorm:"index" json:"blockCid"` // Block CID where proof was included + Height int64 `gorm:"index" json:"height"` // Block height + Method string `json:"method"` // Proof method/algorithm + Verified bool `json:"verified"` // Whether proof was successfully verified + + // Metadata + SectorID *uint64 `json:"sectorId"` + Provider string `gorm:"index" json:"provider"` // Storage Provider ID + ErrorMsg string `json:"errorMessage,omitempty"` +} + +// Create migration for proofs table +func _202507091738_create_proofs() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "202507091738", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&Proof{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable(&Proof{}) + }, + } +} diff --git a/migrate/migrations/migrations.go b/migrate/migrations/migrations.go index e82cbd52..6cb82f6a 100644 --- a/migrate/migrations/migrations.go +++ b/migrate/migrations/migrations.go @@ -11,5 +11,6 @@ func GetMigrations() []*gormigrate.Migration { _202505010840WalletActorID(), _202506240815_create_notifications(), _202506240816_create_deal_templates(), + _202507091738_create_proofs(), } } diff --git a/model/replication.go b/model/replication.go index 9a2e69e2..f006c737 100644 --- a/model/replication.go +++ b/model/replication.go @@ -109,6 +109,36 @@ func (d Deal) Key() string { return fmt.Sprintf("%s-%s-%s-%d-%d", d.ClientActorID, d.Provider, d.PieceCID.String(), d.StartEpoch, d.EndEpoch) } +type ProofType string + +const ( + ProofOfReplication ProofType = "replication" + ProofOfSpacetime ProofType = "spacetime" +) + +type Proof struct { + ID uint64 `gorm:"primaryKey" json:"id"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + + // Link to existing deal + DealID *uint64 `gorm:"index" json:"dealId"` + Deal *Deal `gorm:"foreignKey:DealID;constraint:OnDelete:CASCADE" json:"deal,omitempty"` + + // Proof details + ProofType ProofType `gorm:"index" json:"proofType"` + MessageID string `gorm:"index" json:"messageId"` // Message CID + BlockCID string `gorm:"index" json:"blockCid"` // Block CID where proof was included + Height int64 `gorm:"index" json:"height"` // Block height + Method string `json:"method"` // Proof method/algorithm + Verified bool `json:"verified"` // Whether proof was successfully verified + + // Metadata + SectorID *uint64 `json:"sectorId"` + Provider string `gorm:"index" json:"provider"` // Storage Provider ID + ErrorMsg string `json:"errorMessage,omitempty"` +} + type ScheduleID uint32 type Schedule struct {